00000800:00000200:2.1F:1788201754.538699:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201754.538708:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1788201754.539100:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201754.539102:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1788201754.540041:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201754.540043:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1788201754.540241:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201754.540243:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1788201759.977157:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201759.977577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201759.977691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201759.977816:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201765.097020:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201765.097381:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201765.097610:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201765.097790:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201770.217251:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201770.217254:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 4, 2 secs later) 00000800:00000200:2.1:1788201770.218009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201770.218012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 4, 2 secs later) 00000800:00000200:2.1:1788201770.218347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201770.218350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 4, 2 secs later) 00000800:00000200:2.1:1788201770.218656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201770.218658:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 4, 2 secs later) 00000800:00000200:2.1:1788201775.337043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201775.337354:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201775.337559:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201775.337832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201780.457205:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201780.457213:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 6 secs later) 00000800:00000200:2.1:1788201780.457860:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201780.457863:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 6 secs later) 00000800:00000200:2.1:1788201780.458223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201780.458224:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 6 secs later) 00000800:00000200:2.1:1788201780.458588:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201780.458590:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 6 secs later) 00000800:00000200:2.1:1788201785.577079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201785.577088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 1 secs later) 00000800:00000200:2.1:1788201785.577663:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201785.577664:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 1 secs later) 00000800:00000200:2.1:1788201785.577875:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201785.577876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 1 secs later) 00000800:00000200:2.1:1788201785.578072:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201785.578074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 8, 1 secs later) 00000800:00000200:2.1:1788201790.697160:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201790.697529:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201790.697752:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201790.697940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201795.817332:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201795.817339:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 14 secs later) 00000800:00000200:2.1:1788201795.818122:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201795.818149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 14 secs later) 00000800:00000200:2.1:1788201795.818536:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201795.818539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 14 secs later) 00000800:00000200:2.1:1788201795.818886:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201795.818888:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 14 secs later) 00000800:00000200:2.1:1788201800.937057:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201800.937060:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 9 secs later) 00000800:00000200:2.1:1788201800.937637:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201800.937639:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 9 secs later) 00000800:00000200:2.1:1788201800.937840:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201800.937842:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 9 secs later) 00000800:00000200:2.1:1788201800.938095:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201800.938097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 9 secs later) 00000800:00000200:2.1:1788201806.057234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201806.057237:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 3 secs later) 00000800:00000200:2.1:1788201806.058103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201806.058105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 3 secs later) 00000800:00000200:2.1:1788201806.058310:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201806.058311:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 3 secs later) 00000800:00000200:2.1:1788201806.058585:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201806.058586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 16, 3 secs later) 00000800:00000200:0.1F:1788201811.177016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201811.177353:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201811.177516:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201811.177688:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201816.297482:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201816.297490:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 30 secs later) 00000800:00000200:2.1:1788201816.298517:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201816.298526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 30 secs later) 00000800:00000200:2.1:1788201816.298831:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201816.298832:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 30 secs later) 00000800:00000200:2.1:1788201816.299109:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201816.299110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 30 secs later) 00000800:00000200:2.1:1788201821.417314:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201821.417327:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 25 secs later) 00000800:00000200:2.1:1788201821.418046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201821.418049:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 25 secs later) 00000800:00000200:2.1:1788201821.418479:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201821.418482:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 25 secs later) 00000800:00000200:2.1:1788201821.418784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201821.418786:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 25 secs later) 00000800:00000200:2.1:1788201826.537305:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201826.537308:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 20 secs later) 00000800:00000200:2.1:1788201826.538123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201826.538124:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 20 secs later) 00000800:00000200:2.1:1788201826.538358:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201826.538359:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 20 secs later) 00000800:00000200:2.1:1788201826.538643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201826.538645:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 20 secs later) 00000800:00000200:2.1:1788201831.657269:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201831.657280:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 15 secs later) 00000800:00000200:2.1:1788201831.658263:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201831.658268:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 15 secs later) 00000800:00000200:2.1:1788201831.658498:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201831.658500:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 15 secs later) 00000800:00000200:2.1:1788201831.658834:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201831.658836:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 15 secs later) 00000800:00000200:2.1:1788201836.777240:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201836.777246:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 10 secs later) 00000800:00000200:2.1:1788201836.778087:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201836.778088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 10 secs later) 00000800:00000200:2.1:1788201836.778295:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201836.778297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 10 secs later) 00000800:00000200:2.1:1788201836.778535:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201836.778537:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 10 secs later) 00000800:00000200:2.1:1788201841.896999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201841.897004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 5 secs later) 00000800:00000200:2.1:1788201841.897638:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201841.897641:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 5 secs later) 00000800:00000200:2.1:1788201841.898035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201841.898037:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 5 secs later) 00000800:00000200:2.1:1788201841.898237:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201841.898238:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 32, 5 secs later) 00000800:00000200:2.1:1788201847.016976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201847.017338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201847.017614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201847.017801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201852.136978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201852.136986:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201852.137778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201852.137781:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201852.138103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201852.138105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201852.138416:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201852.138417:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788201857.257086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201857.257092:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201857.257559:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201857.257561:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201857.257835:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201857.257838:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201857.258094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201857.258096:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201862.377007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201862.377013:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201862.377598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201862.377600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201862.377846:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201862.377848:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201862.378067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201862.378069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201867.496991:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201867.496994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788201867.497546:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201867.497548:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788201867.497802:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201867.497804:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788201867.498097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201867.498099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788201872.617271:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201872.617282:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788201872.618055:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201872.618057:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788201872.618310:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201872.618312:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788201872.618544:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201872.618546:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788201877.736957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201877.736968:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788201877.737567:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201877.737574:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788201877.738050:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201877.738052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788201877.738279:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201877.738280:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00010000:00000010:0.1:1788201882.034390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307194600. 00010000:00000010:0.1:1788201882.034395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310d82800. 00010000:00000010:0.1:1788201882.034398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2700. 00010000:00000010:0.1:1788201882.034401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309922e00. 00010000:00000010:0.1:1788201882.034403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56400. 00010000:00000010:0.1:1788201882.034405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31046ea00. 00010000:00000010:0.1:1788201882.034407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c310243000. 00010000:00000010:0.1:1788201882.034415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426400. 00010000:00000010:0.1:1788201882.034417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c310243e00. 00010000:00000010:0.1:1788201882.034419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427200. 00010000:00000010:0.1:1788201882.034421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56d00. 00010000:00000010:0.1:1788201882.034423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c300030e00. 00010000:00000010:0.1:1788201882.034424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56200. 00010000:00000010:0.1:1788201882.034426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c300031c00. 00010000:00000010:0.1:1788201882.034428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c310243900. 00010000:00000010:0.1:1788201882.034430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427400. 00010000:00000010:0.1:1788201882.034432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307194900. 00010000:00000010:0.1:1788201882.034434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310d83800. 00010000:00000010:0.1:1788201882.034435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307ee6a00. 00010000:00000010:0.1:1788201882.034437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059ae00. 00010000:00000010:0.1:1788201882.034439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c319254500. 00010000:00000010:0.1:1788201882.034441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059a600. 00010000:00000010:0.1:1788201882.034443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c321578b00. 00010000:00000010:0.1:1788201882.034445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059a800. 00010000:00000010:0.1:1788201882.034446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c321578900. 00010000:00000010:0.1:1788201882.034448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059bc00. 00010000:00000010:0.1:1788201882.034450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c31b544000. 00010000:00000010:0.1:1788201882.034452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059b800. 00010000:00000010:0.1:1788201882.034453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c319254600. 00010000:00000010:0.1:1788201882.034455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059a200. 00010000:00000010:0.1:1788201882.034457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3113a6f00. 00010000:00000010:0.1:1788201882.034458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426c00. 00000800:00000200:2.1:1788201882.857011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201882.857017:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788201882.857517:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201882.857526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788201882.857753:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201882.857755:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788201882.857970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201882.857974:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788201887.977339:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201887.977345:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788201887.978063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201887.978065:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788201887.978315:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201887.978317:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788201887.978581:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201887.978582:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788201893.097291:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201893.097299:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201893.098096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201893.098098:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201893.098340:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201893.098342:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201893.098595:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201893.098597:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201898.217119:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201898.217130:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201898.217659:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201898.217661:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201898.217867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201898.217869:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201898.218131:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201898.218133:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201903.336935:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201903.336939:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201903.337572:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201903.337574:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201903.337800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201903.337802:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201903.338097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201903.338099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201908.457041:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201908.457045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201908.457574:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201908.457575:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201908.457805:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201908.457807:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201908.457999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201908.458000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201913.577018:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201913.577418:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201913.577584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201913.577777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201918.697074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201918.697080:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788201918.697754:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201918.697756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788201918.698107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201918.698109:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788201918.698349:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201918.698352:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788201923.816955:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201923.816960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201923.817481:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201923.817483:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201923.817693:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201923.817694:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201923.817891:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201923.817893:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788201928.937261:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201928.937273:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201928.937993:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201928.937996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201928.938282:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201928.938285:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201928.938527:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201928.938529:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788201934.057280:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201934.057286:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788201934.058021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201934.058024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788201934.058346:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201934.058349:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788201934.058667:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201934.058669:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788201939.177616:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201939.177624:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788201939.178773:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201939.178778:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788201939.179164:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201939.179167:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788201939.179619:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201939.179622:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788201944.297156:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201944.297164:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788201944.297596:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201944.297598:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788201944.297822:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201944.297824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788201944.298031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201944.298033:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788201949.416967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201949.416971:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788201949.417543:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201949.417545:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788201949.417801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201949.417803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788201949.417999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201949.418001:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788201954.537088:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201954.537097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788201954.537743:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201954.537745:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788201954.537956:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201954.537957:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788201954.538193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201954.538196:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788201959.656984:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201959.656990:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201959.657565:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201959.657567:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201959.657794:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201959.657796:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201959.657996:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201959.657997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788201964.777063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201964.777068:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201964.777616:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201964.777618:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201964.777933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201964.777935:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201964.778131:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201964.778133:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788201969.897296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201969.897301:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201969.898124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201969.898127:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201969.898485:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201969.898488:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201969.898832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201969.898835:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788201975.017144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201975.017148:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201975.017815:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201975.017818:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201975.018060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201975.018062:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201975.018353:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201975.018355:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788201980.137043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201980.137447:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201980.137614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788201980.137790:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201985.257621:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201985.257627:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201985.258107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201985.258110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201985.258559:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201985.258562:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201985.258950:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201985.258953:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788201990.377407:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201990.377417:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788201990.377979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201990.377981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788201990.378220:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201990.378222:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788201990.378469:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201990.378471:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788201995.497169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201995.497175:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788201995.497633:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201995.497635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788201995.497864:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201995.497866:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788201995.498103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788201995.498105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202000.617296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202000.617302:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202000.617948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202000.617950:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202000.618156:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202000.618158:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202000.618348:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202000.618350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202005.737139:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202005.737146:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202005.737602:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202005.737604:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202005.737824:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202005.737826:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202005.738113:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202005.738114:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202010.856981:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202010.856985:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202010.857513:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202010.857516:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202010.857811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202010.857813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202010.858017:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202010.858019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202015.977206:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202015.977209:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202015.977778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202015.977780:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202015.977991:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202015.977993:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202015.978193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202015.978195:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202021.097334:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202021.097346:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202021.097956:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202021.097958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202021.098182:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202021.098184:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202021.098460:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202021.098463:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202026.217240:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202026.217247:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202026.217836:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202026.217838:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202026.218082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202026.218084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202026.218362:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202026.218380:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202031.337449:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202031.337455:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202031.338211:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202031.338214:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202031.338468:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202031.338470:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202031.338721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202031.338723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202036.456974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202036.456978:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202036.457529:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202036.457530:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202036.457734:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202036.457735:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202036.457930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202036.457932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202041.577168:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202041.577183:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202041.577697:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202041.577703:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202041.577959:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202041.577961:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202041.578186:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202041.578190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202046.697131:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202046.697532:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202046.697877:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202046.698068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202051.816922:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202051.816926:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202051.817424:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202051.817427:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202051.817761:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202051.817763:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202051.818066:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202051.818068:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202056.937092:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202056.937097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202056.937736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202056.937739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202056.938073:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202056.938074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202056.938413:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202056.938415:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202062.057198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202062.057204:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202062.057928:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202062.057931:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202062.058254:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202062.058257:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202062.058617:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202062.058620:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202067.177135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202067.177139:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202067.177708:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202067.177709:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202067.177915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202067.177916:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202067.178119:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202067.178122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202072.297158:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202072.297162:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202072.297751:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202072.297753:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202072.297961:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202072.297963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202072.298175:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202072.298177:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202077.417125:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202077.417128:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202077.417841:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202077.417843:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202077.418039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202077.418041:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202077.418263:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202077.418265:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202082.537126:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202082.537132:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202082.537995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202082.537996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202082.538202:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202082.538203:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202082.538474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202082.538477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202087.656952:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202087.656958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202087.657477:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202087.657479:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202087.657687:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202087.657688:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202087.657858:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202087.657860:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202092.777113:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202092.777119:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202092.777614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202092.777617:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202092.777900:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202092.777902:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202092.778199:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202092.778201:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202097.897092:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202097.897099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202097.897591:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202097.897593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202097.897792:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202097.897795:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202097.898007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202097.898008:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202103.017149:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202103.017159:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202103.018040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202103.018042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202103.018314:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202103.018316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202103.018518:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202103.018520:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202108.137304:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202108.137316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202108.138264:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202108.138266:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202108.138568:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202108.138570:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202108.138877:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202108.138878:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202113.257136:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202113.257696:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202113.257905:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202113.258068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202118.377648:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202118.377652:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202118.378085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202118.378088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202118.378404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202118.378406:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202118.378666:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202118.378668:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202123.497026:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202123.497029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202123.497694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202123.497697:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202123.498069:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202123.498070:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202123.498262:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202123.498263:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202128.617429:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202128.617432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202128.617908:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202128.617917:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202128.618261:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202128.618264:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202128.618654:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202128.618657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202133.736945:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202133.736951:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202133.737539:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202133.737541:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202133.737742:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202133.737744:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202133.737940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202133.737941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202138.857050:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202138.857056:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202138.857565:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202138.857567:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202138.857792:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202138.857793:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202138.857986:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202138.857988:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202143.977338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202143.977350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202143.978269:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202143.978272:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202143.978629:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202143.978632:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202143.978953:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202143.978955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202149.097075:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202149.097082:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202149.097548:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202149.097550:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202149.097734:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202149.097736:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202149.097908:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202149.097910:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202154.217093:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202154.217096:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202154.217564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202154.217569:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202154.217772:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202154.217774:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202154.218045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202154.218047:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202159.337046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202159.337055:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202159.337575:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202159.337577:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202159.337779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202159.337781:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202159.338068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202159.338070:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202164.457044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202164.457048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202164.457567:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202164.457570:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202164.457876:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202164.457878:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202164.458124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202164.458126:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202169.577137:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202169.577142:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202169.577847:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202169.577853:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202169.578085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202169.578087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202169.578340:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202169.578342:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202174.697176:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202174.697183:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202174.697728:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202174.697731:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202174.698049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202174.698052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202174.698388:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202174.698390:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202179.817292:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202179.817863:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202179.818303:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202179.818517:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202184.937033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202184.937040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202184.937629:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202184.937631:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202184.937901:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202184.937904:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202184.938183:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202184.938185:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202190.057024:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202190.057028:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202190.057475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202190.057477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202190.057696:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202190.057699:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202190.057904:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202190.057906:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788202195.177055:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202195.177060:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202195.177479:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202195.177481:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202195.177644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202195.177646:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202195.177800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202195.177801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202200.296955:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202200.296959:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202200.297343:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202200.297345:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202200.297571:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202200.297572:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202200.297776:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202200.297777:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202205.416944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202205.416948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202205.417422:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202205.417423:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202205.417672:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202205.417674:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202205.417852:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202205.417854:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202210.536970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202210.536976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202210.537436:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202210.537438:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202210.537620:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202210.537622:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202210.537792:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202210.537794:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202215.657109:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202215.657117:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202215.657678:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202215.657680:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202215.657948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202215.657949:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202215.658205:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202215.658208:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202220.777213:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202220.777217:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202220.778195:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202220.778203:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202220.778541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202220.778543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202220.778773:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202220.778775:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202225.897327:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202225.897334:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202225.898132:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202225.898135:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202225.898508:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202225.898509:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202225.898739:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202225.898741:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202231.017029:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202231.017036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202231.017529:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202231.017532:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202231.017835:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202231.017837:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202231.018120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202231.018122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202236.136990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202236.136994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202236.137598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202236.137600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202236.137905:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202236.137907:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202236.138198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202236.138201:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202241.256963:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202241.256967:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202241.257497:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202241.257498:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202241.257685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202241.257687:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202241.257851:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202241.257853:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202246.377082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202246.377393:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202246.377541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202246.377679:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202251.497223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202251.497230:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202251.497945:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202251.497948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202251.498281:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202251.498283:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202251.498640:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202251.498643:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202256.617102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202256.617110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202256.617617:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202256.617619:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202256.617808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202256.617810:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202256.617984:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202256.617986:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202261.737085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202261.737091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202261.737670:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202261.737672:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202261.737893:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202261.737899:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202261.738166:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202261.738168:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202266.857393:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202266.857399:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202266.857923:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202266.857926:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202266.858294:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202266.858297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202266.858654:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202266.858657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202271.977125:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202271.977132:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202271.977694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202271.977697:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202271.978042:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202271.978045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202271.978351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202271.978354:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202277.096951:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202277.096954:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202277.097593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202277.097595:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202277.097799:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202277.097801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202277.098118:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202277.098121:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202282.217035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202282.217040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202282.217606:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202282.217608:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202282.217940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202282.217942:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202282.218139:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202282.218140:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00010000:00000010:2.1:1788202282.415404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3216e8b00. 00010000:00000010:2.1:1788202282.415410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059ac00. 00010000:00000010:2.1:1788202282.415413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56000. 00010000:00000010:2.1:1788202282.415415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31046e200. 00010000:00000010:2.1:1788202282.415417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307194400. 00010000:00000010:2.1:1788202282.415418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310d82600. 00010000:00000010:2.1:1788202282.415421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3025ac300. 00010000:00000010:2.1:1788202282.415423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426200. 00010000:00000010:2.1:1788202282.415425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2600. 00010000:00000010:2.1:1788202282.415427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309923c00. 00010000:00000010:2.1:1788202282.415429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2f00. 00010000:00000010:2.1:1788202282.415431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309922000. 00010000:00000010:2.1:1788202282.415433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3113a6400. 00010000:00000010:2.1:1788202282.415435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426600. 00010000:00000010:2.1:1788202282.415436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307194800. 00010000:00000010:2.1:1788202282.415438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310d83400. 00010000:00000010:2.1:1788202282.415440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56600. 00010000:00000010:2.1:1788202282.415441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c300031e00. 00010000:00000010:2.1:1788202282.415444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307e56900. 00010000:00000010:2.1:1788202282.415445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c300030200. 00010000:00000010:2.1:1788202282.415447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2c00. 00010000:00000010:2.1:1788202282.415455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309923a00. 00010000:00000010:2.1:1788202282.415457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2800. 00010000:00000010:2.1:1788202282.415459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309923e00. 00010000:00000010:2.1:1788202282.415461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c310243a00. 00010000:00000010:2.1:1788202282.415463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427000. 00010000:00000010:2.1:1788202282.415464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3113a6700. 00010000:00000010:2.1:1788202282.415466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427600. 00010000:00000010:2.1:1788202282.415467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307ee6400. 00010000:00000010:2.1:1788202282.415469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426000. 00010000:00000010:2.1:1788202282.415471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307ee6800. 00010000:00000010:2.1:1788202282.415472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427a00. 00000800:00000200:0.1:1788202287.337333:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202287.337346:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202287.338249:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202287.338251:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202287.338502:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202287.338508:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202287.338756:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202287.338758:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000080:00000010:0.1:1788202289.582455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c307131880. 00000080:00000010:0.1:1788202289.582476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c306af9c80. 00000080:00000010:0.1:1788202289.582479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c307131120. 00000080:00000010:0.1:1788202289.586397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c307ffa480. 00000080:00000010:0.1:1788202289.586411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c307ffad80. 00000080:00000010:0.1:1788202289.586414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff98c307131ae0. 00010000:00000010:0.1:1788202289.586420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3216e8400. 00010000:00000010:0.1:1788202289.586432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059b000. 00010000:00000010:0.1:1788202289.586435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3216e8100. 00010000:00000010:0.1:1788202289.586438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310d9c600. 00010000:00000010:0.1:1788202289.586443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c319254400. 00010000:00000010:0.1:1788202289.586448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309923600. 00010000:00000010:0.1:1788202289.586452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c305ce2b00. 00010000:00000010:0.1:1788202289.586454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309923200. 00010000:00000010:0.1:1788202289.586456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c309922c00. 00010000:00000010:0.1:1788202289.586458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c3113a6200. 00010000:00000010:0.1:1788202289.586461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310427800. 00010000:00000010:0.1:1788202289.586463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c319254a00. 00010000:00000010:0.1:1788202289.586465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059b600. 00010000:00000010:0.1:1788202289.586467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c319254800. 00010000:00000010:0.1:1788202289.586469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c31059ba00. 00010000:00000010:0.1:1788202289.586471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff98c307ee6d00. 00010000:00000010:0.1:1788202289.586473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff98c310426800. 00000800:00000200:0.1:1788202292.457017:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202292.457022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202292.457618:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202292.457620:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202292.457848:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202292.457850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202292.458026:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202292.458027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202297.576957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202297.576965:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202297.577635:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202297.577637:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202297.577826:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202297.577828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202297.578104:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202297.578105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202302.697124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202302.697131:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202302.697709:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202302.697712:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202302.698039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202302.698042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202302.698403:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202302.698406:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202307.817044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202307.817048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202307.817635:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202307.817638:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202307.817882:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202307.817884:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202307.818144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202307.818145:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202312.937431:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202312.937674:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202312.937783:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202312.937923:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202318.057142:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202318.057150:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202318.057736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202318.057739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202318.058029:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202318.058031:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202318.058345:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202318.058347:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202323.177103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202323.177110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202323.177644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202323.177647:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202323.177937:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202323.177939:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202323.178257:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202323.178259:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202328.296960:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202328.296972:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202328.297561:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202328.297563:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202328.297752:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202328.297754:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202328.297948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202328.297950:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202333.417097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202333.417104:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202333.417811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202333.417813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202333.418032:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202333.418034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202333.418313:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202333.418316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202338.537259:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202338.537269:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202338.538135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202338.538137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202338.538398:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202338.538400:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202338.538673:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202338.538675:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202343.657181:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202343.657189:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202343.657845:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202343.657848:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202343.658134:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202343.658137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202343.658395:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202343.658397:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202348.777211:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202348.777240:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202348.777934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202348.777937:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202348.778263:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202348.778266:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202348.778610:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202348.778612:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202353.897080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202353.897085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202353.897657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202353.897660:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202353.897875:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202353.897877:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202353.898079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202353.898081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202359.017013:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202359.017017:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202359.017561:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202359.017562:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202359.017748:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202359.017749:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202359.017922:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202359.017924:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202364.136970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202364.136983:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202364.137548:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202364.137549:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202364.137750:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202364.137753:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202364.137967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202364.137970:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202369.257149:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202369.257154:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202369.257721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202369.257723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202369.257967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202369.257969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202369.258226:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202369.258228:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202374.377250:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202374.377276:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202374.378209:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202374.378212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202374.378481:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202374.378482:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202374.378733:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202374.378736:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202379.497027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202379.497409:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202379.497570:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202379.497735:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202384.617690:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202384.617693:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202384.618070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202384.618073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202384.618419:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202384.618436:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202384.618707:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202384.618709:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202389.736994:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202389.736999:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202389.737613:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202389.737616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202389.737829:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202389.737830:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202389.738014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202389.738016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202394.857025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202394.857034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202394.857587:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202394.857590:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202394.857800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202394.857802:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202394.858021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202394.858024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202399.977042:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202399.977049:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202399.977656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202399.977659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202399.977999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202399.978015:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202399.978352:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202399.978355:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202405.097099:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202405.097106:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202405.097713:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202405.097715:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202405.098017:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202405.098020:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202405.098315:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202405.098318:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202410.216987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202410.216991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202410.217644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202410.217645:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202410.217862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202410.217864:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202410.218150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202410.218153:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202415.337200:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202415.337208:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202415.337886:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202415.337889:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202415.338203:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202415.338214:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202415.338459:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202415.338461:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202420.457129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202420.457137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202420.457698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202420.457701:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202420.457993:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202420.457995:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202420.458328:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202420.458331:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202425.577155:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202425.577158:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202425.577657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202425.577659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202425.577826:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202425.577828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202425.578164:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202425.578167:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202430.697070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202430.697078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202430.697685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202430.697688:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202430.697876:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202430.697893:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202430.698065:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202430.698068:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202435.817333:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202435.817339:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202435.818028:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202435.818031:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202435.818404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202435.818407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202435.818730:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202435.818733:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202440.937094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202440.937099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202440.937771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202440.937773:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202440.938012:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202440.938014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202440.938292:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202440.938293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202446.057076:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202446.057447:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202446.057621:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202446.057802:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202451.177312:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202451.177316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202451.178297:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202451.178300:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202451.178677:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202451.178679:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202451.178908:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202451.178909:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202456.297390:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202456.297392:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202456.298217:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202456.298219:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202456.298466:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202456.298468:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202456.298777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202456.298779:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202461.417011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202461.417016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202461.417644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202461.417647:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202461.417911:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202461.417913:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202461.418228:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202461.418232:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202466.537047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202466.537050:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202466.537612:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202466.537614:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202466.537840:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202466.537842:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202466.538061:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202466.538063:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202471.657100:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202471.657105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202471.657819:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202471.657821:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202471.658064:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202471.658066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202471.658360:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202471.658362:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202476.777011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202476.777029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202476.777644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202476.777647:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202476.777896:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202476.777898:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202476.778196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202476.778199:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202481.897125:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202481.897143:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202481.897597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202481.897598:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202481.897765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202481.897766:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202481.897916:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202481.897917:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202487.017015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202487.017023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202487.017491:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202487.017492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202487.017652:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202487.017653:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202487.017804:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202487.017806:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202492.136948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202492.136951:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202492.137548:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202492.137550:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202492.137722:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202492.137723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202492.137920:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202492.137922:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202497.257080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202497.257099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202497.257695:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202497.257696:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202497.257881:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202497.257883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202497.258083:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202497.258085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202502.377187:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202502.377194:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202502.377841:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202502.377843:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202502.378086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202502.378088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202502.378348:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202502.378350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202507.497044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202507.497048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202507.497563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202507.497565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202507.497754:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202507.497756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202507.497979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202507.497981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202512.617265:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202512.617757:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202512.617942:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202512.618127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202517.737423:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202517.737431:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202517.738089:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202517.738092:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202517.738428:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202517.738431:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202517.738736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202517.738739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202522.857001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202522.857005:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202522.857647:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202522.857649:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202522.857925:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202522.857927:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202522.858196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202522.858198:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202527.977018:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202527.977023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202527.977714:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202527.977718:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202527.977943:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202527.977944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202527.978170:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202527.978172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202533.097101:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202533.097109:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202533.097767:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202533.097770:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202533.098031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202533.098033:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202533.098347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202533.098350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202538.217009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202538.217014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202538.217486:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202538.217488:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202538.217744:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202538.217745:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202538.217975:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202538.217977:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202543.337236:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202543.337245:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202543.337830:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202543.337833:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202543.338196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202543.338199:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202543.338553:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202543.338556:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202548.457136:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202548.457144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202548.457722:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202548.457725:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202548.457996:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202548.458009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202548.458228:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202548.458229:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202553.577273:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202553.577279:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202553.577734:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202553.577737:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202553.577944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202553.577945:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202553.578161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202553.578163:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202558.697114:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202558.697122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202558.697646:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202558.697649:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202558.698004:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202558.698007:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202558.698308:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202558.698310:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202563.817088:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202563.817095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202563.817589:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202563.817591:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202563.817774:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202563.817776:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202563.818014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202563.818016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202568.937087:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202568.937095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202568.937627:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202568.937629:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202568.937847:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202568.937850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202568.938084:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202568.938086:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202574.057094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202574.057096:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202574.057792:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202574.057794:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202574.058033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202574.058035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202574.058261:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202574.058264:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202579.176974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202579.177272:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202579.177472:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202579.177655:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202584.297145:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202584.297153:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202584.297884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202584.297887:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202584.298220:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202584.298223:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202584.298685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202584.298698:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202589.416987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202589.416991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202589.417546:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202589.417548:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202589.417735:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202589.417736:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202589.417915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202589.417916:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202594.537142:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202594.537149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202594.537557:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202594.537559:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202594.537859:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202594.537861:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202594.538153:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202594.538155:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202599.657038:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202599.657042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202599.657633:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202599.657635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202599.657941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202599.657944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202599.658259:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202599.658269:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202604.777072:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202604.777091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202604.777585:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202604.777586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202604.777808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202604.777810:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202604.777990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202604.777992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202609.897274:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202609.897282:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202609.897895:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202609.897899:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202609.898161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202609.898165:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202609.898435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202609.898438:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202615.017163:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202615.017169:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202615.017553:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202615.017555:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202615.017737:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202615.017739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202615.017934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202615.017944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202620.137062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202620.137066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202620.137645:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202620.137647:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202620.137934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202620.137937:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202620.138209:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202620.138211:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202625.257094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202625.257101:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202625.257780:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202625.257783:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202625.258080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202625.258083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202625.258435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202625.258438:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202630.377331:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202630.377338:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202630.378003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202630.378005:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202630.378218:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202630.378220:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202630.378430:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202630.378447:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202635.497103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202635.497108:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202635.497661:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202635.497664:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202635.497987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202635.497990:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202635.498296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202635.498299:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202640.617198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202640.617206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202640.617618:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202640.617620:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202640.617815:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202640.617817:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202640.618032:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202640.618034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202645.737267:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202645.737765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202645.737943:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202645.738125:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202650.857074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202650.857079:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202650.857753:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202650.857756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202650.858053:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202650.858056:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202650.858328:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202650.858331:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202655.977276:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202655.977288:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202655.978346:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202655.978349:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202655.978629:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202655.978631:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202655.978894:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202655.978896:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202661.097300:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202661.097306:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202661.098128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202661.098130:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202661.098402:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202661.098404:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202661.098662:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202661.098664:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202666.217337:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202666.217349:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202666.218241:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202666.218244:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202666.218497:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202666.218499:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202666.218811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202666.218813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202671.337037:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202671.337046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202671.337578:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202671.337580:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202671.337806:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202671.337808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202671.338019:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202671.338021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202676.456936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202676.456939:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202676.457335:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202676.457337:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202676.457601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202676.457603:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202676.457782:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202676.457784:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202681.577011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202681.577016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202681.577650:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202681.577654:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202681.577902:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202681.577903:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202681.578115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202681.578118:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202686.697139:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202686.697145:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202686.697644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202686.697646:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202686.697869:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202686.697871:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202686.698045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202686.698047:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202691.817067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202691.817074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202691.817600:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202691.817603:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202691.817862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202691.817864:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202691.818118:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202691.818120:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202696.937042:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202696.937045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202696.937620:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202696.937622:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202696.937932:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202696.937934:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202696.938229:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202696.938232:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202702.056957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202702.056962:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202702.057540:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202702.057542:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202702.057764:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202702.057765:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202702.058004:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202702.058006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202707.177068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202707.177079:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202707.177703:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202707.177705:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202707.177976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202707.177979:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202707.178266:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202707.178269:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202712.296969:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202712.297278:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202712.297439:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202712.297584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202717.417277:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202717.417281:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202717.417654:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202717.417657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202717.417918:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202717.417920:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202717.418204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202717.418207:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202722.537055:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202722.537069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202722.537562:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202722.537565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202722.537871:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202722.537874:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202722.538191:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202722.538192:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202727.656958:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202727.656961:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202727.657572:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202727.657574:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202727.657760:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202727.657762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202727.658075:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202727.658078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202732.776978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202732.776983:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202732.777610:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202732.777613:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202732.777823:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202732.777825:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202732.778057:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202732.778059:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202737.897060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202737.897074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202737.897633:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202737.897635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202737.897850:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202737.897852:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202737.898081:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202737.898084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202743.017062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202743.017066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202743.017743:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202743.017745:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202743.017985:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202743.017987:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202743.018229:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202743.018231:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202748.137279:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202748.137286:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202748.138223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202748.138224:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202748.138474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202748.138476:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202748.138745:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202748.138747:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202753.257009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202753.257022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202753.257631:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202753.257634:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202753.257869:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202753.257871:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202753.258154:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202753.258156:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202758.377142:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202758.377149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202758.377586:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202758.377587:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202758.377820:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202758.377822:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202758.378038:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202758.378041:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202763.496960:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202763.496964:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202763.497490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202763.497492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202763.497704:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202763.497706:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202763.497894:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202763.497896:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202768.617315:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202768.617325:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202768.618178:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202768.618181:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202768.618502:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202768.618505:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202768.618805:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202768.618808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202773.737317:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202773.737330:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202773.738183:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202773.738187:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202773.738524:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202773.738527:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202773.738835:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202773.738839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202778.857194:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202778.857653:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202778.857898:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202778.858080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202783.977137:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202783.977145:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202783.977874:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202783.977876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202783.978201:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202783.978203:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202783.978495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202783.978497:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202789.097127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202789.097134:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202789.097703:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202789.097706:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202789.098051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202789.098054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202789.098347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202789.098349:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788202794.217144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202794.217150:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202794.217723:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202794.217724:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202794.217937:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202794.217939:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202794.218185:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202794.218187:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788202799.337234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202799.337246:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202799.337936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202799.337938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202799.338193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202799.338196:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202799.338527:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202799.338530:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788202804.457316:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202804.457322:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202804.458052:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202804.458054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202804.458242:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202804.458244:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202804.458450:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202804.458452:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788202809.577459:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202809.577467:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202809.578078:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202809.578081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202809.578425:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202809.578428:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202809.578766:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202809.578769:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202814.697336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202814.697346:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202814.698129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202814.698131:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202814.698324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202814.698326:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202814.698584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202814.698586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202819.817010:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202819.817019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202819.817603:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202819.817605:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202819.817799:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202819.817801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202819.817980:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202819.817983:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202824.937439:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202824.937446:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202824.938055:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202824.938057:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202824.938239:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202824.938241:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202824.938580:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202824.938582:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202830.057347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202830.057355:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202830.058082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202830.058084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202830.058441:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202830.058444:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202830.058736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202830.058738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788202835.177148:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202835.177155:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202835.177803:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202835.177806:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202835.178144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202835.178147:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202835.178474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202835.178477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788202840.297216:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202840.297222:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202840.298152:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202840.298159:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202840.298592:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202840.298594:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202840.298818:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202840.298821:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788202845.417249:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202845.417728:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202845.418249:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202845.418450:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202850.537238:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202850.537244:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202850.537860:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202850.537874:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202850.538171:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202850.538174:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202850.538514:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202850.538516:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202855.657137:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202855.657144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202855.657623:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202855.657625:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202855.657931:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202855.657932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202855.658262:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202855.658265:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202860.777035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202860.777039:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202860.777748:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202860.777749:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202860.777972:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202860.777974:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202860.778338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202860.778340:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202865.897246:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202865.897251:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202865.898169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202865.898172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202865.898837:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202865.898839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202865.899068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202865.899069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202871.017003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202871.017008:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202871.017539:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202871.017542:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202871.017801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202871.017803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202871.018063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202871.018065:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202876.137036:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202876.137042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202876.137578:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202876.137580:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202876.137785:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202876.137786:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202876.137969:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202876.137971:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788202881.256917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202881.256923:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202881.257379:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202881.257380:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202881.257575:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202881.257577:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202881.257756:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202881.257757:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788202886.376939:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202886.376946:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202886.377591:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202886.377593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202886.377857:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202886.377859:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202886.378091:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202886.378093:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788202891.497216:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202891.497229:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202891.498080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202891.498095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202891.498464:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202891.498467:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202891.498779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202891.498782:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202896.617294:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202896.617302:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202896.618068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202896.618071:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202896.618275:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202896.618277:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202896.618497:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202896.618499:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202901.736959:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202901.736963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202901.737402:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202901.737403:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202901.737593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202901.737595:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202901.737781:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202901.737783:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202906.857002:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202906.857006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202906.857476:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202906.857488:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202906.857668:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202906.857669:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202906.857823:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202906.857824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202911.977128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202911.977529:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202911.977799:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202911.977990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202917.097343:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202917.097346:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202917.097687:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202917.097689:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202917.097970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202917.097973:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202917.098223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202917.098225:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788202922.217135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202922.217140:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202922.217688:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202922.217691:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202922.217977:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202922.217980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202922.218144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202922.218146:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788202927.337245:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202927.337255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202927.337759:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202927.337762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202927.337971:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202927.337973:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202927.338161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202927.338163:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788202932.457105:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202932.457112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202932.457580:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202932.457582:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202932.457859:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202932.457860:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202932.458030:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202932.458032:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788202937.577459:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202937.577469:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202937.578095:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202937.578097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202937.578351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202937.578354:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202937.578647:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202937.578649:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788202942.696990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202942.696994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202942.697618:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202942.697619:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202942.697814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202942.697816:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202942.697991:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202942.697992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788202947.817060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202947.817067:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202947.817691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202947.817693:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202947.817915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202947.817917:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788202947.818103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202947.818105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202952.936984:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202952.936991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202952.937608:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202952.937610:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202952.937797:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202952.937799:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202952.938015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202952.938017:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202958.057040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202958.057044:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202958.057633:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202958.057635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202958.057966:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202958.057968:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202958.058207:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202958.058209:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788202963.177009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202963.177014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202963.177546:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202963.177547:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202963.177742:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202963.177744:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202963.177949:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202963.177951:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788202968.297204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202968.297212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202968.297599:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202968.297601:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202968.297807:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202968.297809:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202968.297990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202968.297992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788202973.417213:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202973.417221:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202973.418170:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202973.418178:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202973.418489:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202973.418491:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788202973.418770:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788202973.418771:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202978.537287:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202978.537895:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202978.538146:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202978.538291:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202983.657435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202983.657439:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202983.657897:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202983.657900:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202983.658237:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202983.658241:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202983.658771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202983.658773:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202988.777115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202988.777121:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202988.777657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202988.777666:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202988.777908:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202988.777911:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202988.778192:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202988.778194:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202993.897062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202993.897066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202993.897601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202993.897604:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202993.897860:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202993.897862:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202993.898076:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202993.898078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202999.017107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202999.017113:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202999.017781:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202999.017783:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202999.018011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202999.018013:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202999.018260:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788202999.018262:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203004.137054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203004.137057:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203004.137589:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203004.137603:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203004.137831:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203004.137832:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203004.138020:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203004.138022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203009.257140:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203009.257148:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203009.257815:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203009.257818:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203009.258129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203009.258131:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203009.258390:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203009.258391:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203014.377239:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203014.377243:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203014.378120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203014.378127:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203014.378436:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203014.378438:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203014.378670:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203014.378672:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203019.497101:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203019.497106:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203019.497718:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203019.497720:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203019.497941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203019.497943:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203019.498202:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203019.498203:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203024.617049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203024.617054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203024.617699:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203024.617701:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203024.617914:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203024.617916:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203024.618196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203024.618198:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203029.737075:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203029.737084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203029.737530:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203029.737532:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203029.737737:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203029.737739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203029.737919:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203029.737920:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203034.857088:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203034.857093:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203034.857784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203034.857801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203034.858102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203034.858105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203034.858419:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203034.858423:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203039.977037:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203039.977043:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203039.977540:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203039.977541:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203039.977767:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203039.977769:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203039.977947:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203039.977949:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203045.096964:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203045.097243:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203045.097454:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203045.097601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203050.217295:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203050.217297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203050.217698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203050.217700:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203050.217957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203050.217960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203050.218317:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203050.218319:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203055.336974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203055.336978:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203055.337410:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203055.337412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203055.337604:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203055.337605:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203055.337780:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203055.337782:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203060.457102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203060.457110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203060.457637:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203060.457639:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203060.457825:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203060.457827:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203060.458003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203060.458004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203065.577003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203065.577011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203065.577541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203065.577543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203065.577759:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203065.577761:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203065.577957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203065.577960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203070.696954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203070.696960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203070.697457:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203070.697459:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203070.697761:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203070.697764:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203070.698060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203070.698062:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203075.817104:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203075.817111:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203075.817623:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203075.817625:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203075.817810:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203075.817811:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203075.818014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203075.818016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203080.937037:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203080.937040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203080.937688:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203080.937694:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203080.937941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203080.937943:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203080.938166:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203080.938168:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203086.057080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203086.057088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203086.057659:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203086.057661:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203086.057930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203086.057933:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203086.058257:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203086.058260:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203091.177008:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203091.177016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203091.177512:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203091.177514:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203091.177721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203091.177723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203091.177964:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203091.177967:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203096.297006:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203096.297012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203096.297519:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203096.297522:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203096.297793:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203096.297795:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203096.297973:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203096.297974:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203101.417046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203101.417052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203101.417751:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203101.417756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203101.417999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203101.418001:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203101.418276:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203101.418279:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203106.537295:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203106.537313:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203106.538012:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203106.538015:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203106.538345:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203106.538348:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203106.538709:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203106.538712:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203111.657054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203111.657350:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203111.657539:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203111.657703:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203116.777074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203116.777079:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203116.777770:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203116.777773:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203116.778089:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203116.778092:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203116.778397:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203116.778398:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203121.897140:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203121.897149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203121.897635:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203121.897637:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203121.897823:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203121.897825:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203121.897998:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203121.897999:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203127.017039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203127.017043:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203127.017597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203127.017600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203127.017817:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203127.017818:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203127.017994:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203127.017995:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203132.137418:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203132.137432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203132.138047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203132.138050:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203132.138336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203132.138350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203132.138556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203132.138558:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203137.257025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203137.257030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203137.257540:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203137.257542:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203137.257733:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203137.257735:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203137.257930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203137.257932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203142.377086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203142.377094:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203142.377794:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203142.377797:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203142.378092:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203142.378095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203142.378417:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203142.378419:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203147.496886:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203147.496891:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203147.497252:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203147.497254:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203147.497462:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203147.497464:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203147.497640:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203147.497641:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203152.617005:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203152.617012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203152.617549:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203152.617551:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203152.617765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203152.617767:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203152.617940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203152.617942:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203157.737066:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203157.737072:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203157.737494:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203157.737497:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203157.737721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203157.737723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203157.737941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203157.737944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203162.856954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203162.856958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203162.857339:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203162.857341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203162.857586:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203162.857588:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203162.857775:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203162.857778:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203167.977289:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203167.977311:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203167.978026:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203167.978029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203167.978404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203167.978407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203167.978723:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203167.978726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203173.097519:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203173.097530:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203173.097912:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203173.097914:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203173.098098:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203173.098101:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203173.098297:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203173.098299:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203178.216940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203178.217229:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203178.217397:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203178.217522:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203183.337115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203183.337122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203183.337741:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203183.337743:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203183.338000:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203183.338003:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203183.338319:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203183.338321:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203188.457057:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203188.457061:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203188.457597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203188.457599:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203188.457785:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203188.457787:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203188.457975:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203188.457977:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203193.577148:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203193.577152:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203193.577583:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203193.577586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203193.577884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203193.577887:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203193.578174:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203193.578176:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203198.697059:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203198.697065:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203198.697656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203198.697658:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203198.697950:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203198.697953:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203198.698222:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203198.698224:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203203.817025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203203.817030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203203.817407:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203203.817409:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203203.817604:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203203.817606:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203203.817784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203203.817785:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203208.937019:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203208.937023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203208.937508:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203208.937510:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203208.937758:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203208.937760:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203208.938009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203208.938011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203214.056795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203214.056799:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203214.057110:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203214.057111:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203214.057268:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203214.057269:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203214.057440:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203214.057442:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203219.176957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203219.176962:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203219.177287:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203219.177289:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203219.177522:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203219.177523:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203219.177679:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203219.177680:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203224.297086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203224.297091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203224.297563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203224.297564:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203224.297770:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203224.297772:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203224.297959:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203224.297961:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203229.417146:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203229.417153:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203229.417745:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203229.417748:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203229.418027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203229.418029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203229.418268:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203229.418269:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203234.537227:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203234.537231:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203234.538104:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203234.538107:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203234.538398:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203234.538400:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203234.538726:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203234.538727:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203239.657129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203239.657137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203239.657578:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203239.657581:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203239.657789:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203239.657791:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203239.657980:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203239.657981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203244.777010:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203244.777430:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203244.777577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203244.777711:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203249.897228:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203249.897233:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203249.899060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203249.899062:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203249.899500:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203249.899503:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203249.899818:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203249.899820:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203255.017016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203255.017019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203255.017440:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203255.017442:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203255.017680:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203255.017682:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203255.017881:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203255.017883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203260.137179:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203260.137183:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203260.137652:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203260.137655:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203260.137954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203260.137957:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203260.138226:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203260.138229:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203265.257123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203265.257127:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203265.257711:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203265.257713:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203265.257956:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203265.257960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203265.258165:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203265.258167:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203270.377173:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203270.377182:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203270.377800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203270.377803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203270.378152:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203270.378156:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203270.378507:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203270.378509:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203275.497296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203275.497301:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203275.498132:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203275.498134:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203275.498329:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203275.498331:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203275.498566:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203275.498569:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203280.617144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203280.617153:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203280.617558:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203280.617560:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203280.617845:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203280.617858:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203280.618126:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203280.618128:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203285.737064:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203285.737072:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203285.737533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203285.737535:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203285.737724:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203285.737726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203285.737898:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203285.737900:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203290.857037:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203290.857041:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203290.857584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203290.857586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203290.857771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203290.857772:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203290.857944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203290.857946:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203295.977085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203295.977089:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203295.977674:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203295.977676:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203295.977866:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203295.977880:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203295.978066:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203295.978067:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203301.097070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203301.097077:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203301.097594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203301.097596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203301.097812:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203301.097813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203301.098016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203301.098018:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203306.217106:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203306.217112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203306.217656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203306.217659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203306.217893:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203306.217895:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203306.218146:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203306.218147:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203311.337144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203311.337551:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203311.337935:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203311.338108:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203316.457480:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203316.457491:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203316.458452:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203316.458454:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203316.458753:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203316.458756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203316.459071:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203316.459074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203321.577197:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203321.577209:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203321.578145:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203321.578147:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203321.578339:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203321.578341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203321.578543:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203321.578544:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203326.697277:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203326.697287:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203326.698233:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203326.698235:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203326.698445:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203326.698460:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203326.698740:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203326.698742:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203331.817046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203331.817051:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203331.817698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203331.817700:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203331.817887:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203331.817889:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203331.818077:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203331.818080:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203336.937140:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203336.937148:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203336.937685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203336.937687:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203336.937998:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203336.938001:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203336.938317:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203336.938319:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203342.057046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203342.057052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203342.057584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203342.057585:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203342.057783:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203342.057790:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203342.057975:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203342.057976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203347.177044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203347.177051:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203347.177591:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203347.177592:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203347.177789:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203347.177791:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203347.178031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203347.178034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203352.297147:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203352.297154:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203352.297643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203352.297646:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203352.297850:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203352.297852:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203352.298027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203352.298029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203357.416966:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203357.416969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203357.417570:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203357.417572:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203357.417788:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203357.417790:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203357.417968:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203357.417969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203362.537224:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203362.537233:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203362.537915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203362.537917:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203362.538196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203362.538206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203362.538543:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203362.538545:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203367.657297:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203367.657307:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203367.658025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203367.658027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203367.658260:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203367.658262:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203367.658500:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203367.658502:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203372.777324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203372.777331:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203372.778028:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203372.778031:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203372.778284:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203372.778286:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203372.778540:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203372.778543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203377.897198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203377.897518:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203377.897628:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203377.897811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203383.017732:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203383.017738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203383.018189:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203383.018212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203383.018521:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203383.018523:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203383.018979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203383.018982:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203388.137332:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203388.137345:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203388.138035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203388.138037:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203388.138237:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203388.138238:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203388.138512:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203388.138515:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203393.257085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203393.257090:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203393.257725:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203393.257727:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203393.257957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203393.257958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203393.258193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203393.258195:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788203398.377282:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203398.377290:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203398.378217:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203398.378242:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203398.378555:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203398.378556:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203398.378796:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203398.378798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203403.497418:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203403.497426:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203403.497963:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203403.497965:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203403.498154:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203403.498168:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203403.498387:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203403.498389:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203408.617073:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203408.617076:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203408.617660:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203408.617662:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203408.617848:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203408.617850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203408.618104:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203408.618105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203413.736975:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203413.736981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203413.737628:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203413.737631:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203413.737849:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203413.737851:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203413.738026:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203413.738028:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203418.857459:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203418.857472:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203418.858314:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203418.858316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203418.858563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203418.858580:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203418.858802:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203418.858804:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203423.977207:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203423.977216:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203423.977968:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203423.977971:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203423.978258:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203423.978260:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203423.978523:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203423.978525:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203429.097009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203429.097014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788203429.097734:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203429.097736:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788203429.097948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203429.097950:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788203429.098180:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203429.098182:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788203434.217032:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203434.217035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203434.217643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203434.217657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203434.217867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203434.217869:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788203434.218105:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203434.218107:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203439.337290:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203439.337299:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203439.338007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203439.338009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203439.338220:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203439.338221:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203439.338471:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203439.338473:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788203444.457144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203444.457511:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203444.457666:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203444.457831:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203449.577266:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203449.577272:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203449.577941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203449.577944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203449.578268:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203449.578271:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203449.578631:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203449.578634:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203454.697200:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203454.697206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203454.697706:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203454.697708:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203454.697979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203454.697980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203454.698169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203454.698171:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203459.816934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203459.816938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203459.817412:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203459.817414:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203459.817643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203459.817655:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203459.817837:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203459.817839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203464.937115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203464.937120:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203464.937814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203464.937815:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203464.938034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203464.938036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203464.938243:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203464.938245:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203470.057002:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203470.057006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203470.057593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203470.057595:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203470.057783:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203470.057785:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203470.058004:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203470.058006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203475.177417:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203475.177420:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203475.177750:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203475.177752:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203475.177947:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203475.177949:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203475.178165:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203475.178167:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203480.297062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203480.297068:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203480.297577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203480.297580:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203480.297801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203480.297803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203480.298008:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203480.298010:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203485.416995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203485.417001:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203485.417615:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203485.417617:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203485.417827:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203485.417829:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203485.418009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203485.418011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203490.537478:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203490.537493:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203490.538047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203490.538050:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203490.538413:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203490.538416:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203490.538756:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203490.538759:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203495.657056:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203495.657063:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203495.657517:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203495.657519:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203495.657755:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203495.657756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203495.658034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203495.658036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203500.777361:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203500.777392:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203500.778030:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203500.778033:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203500.778253:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203500.778255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203500.778490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203500.778491:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203505.897126:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203505.897132:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203505.897600:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203505.897602:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203505.897800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203505.897802:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203505.897977:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203505.897979:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203511.017150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203511.017691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203511.017925:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203511.018094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203516.137103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203516.137110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203516.137871:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203516.137874:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203516.138186:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203516.138189:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203516.138542:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203516.138544:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203521.257096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203521.257100:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203521.257605:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203521.257608:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203521.257903:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203521.257906:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203521.258161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203521.258176:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203526.376988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203526.376994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203526.377587:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203526.377590:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203526.377836:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203526.377839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203526.378039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203526.378040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203531.497058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203531.497064:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203531.497556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203531.497558:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203531.497778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203531.497780:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203531.498007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203531.498008:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203536.617154:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203536.617162:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203536.617632:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203536.617634:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203536.617822:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203536.617824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203536.618020:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203536.618022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203541.736996:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203541.737000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203541.737336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203541.737338:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203541.737607:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203541.737609:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203541.737795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203541.737797:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203546.856978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203546.856984:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203546.857499:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203546.857502:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203546.857705:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203546.857707:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203546.857905:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203546.857907:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203551.976979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203551.976982:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203551.977672:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203551.977675:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203551.977952:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203551.977955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203551.978190:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203551.978192:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203557.097084:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203557.097088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203557.097533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203557.097535:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203557.097877:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203557.097880:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203557.098080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203557.098081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203562.217146:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203562.217150:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203562.217842:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203562.217844:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203562.218133:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203562.218136:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203562.218581:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203562.218584:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203567.337226:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203567.337234:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203567.337883:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203567.337886:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203567.338223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203567.338226:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203567.338583:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203567.338586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203572.457338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203572.457350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203572.458073:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203572.458076:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203572.458428:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203572.458432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203572.458736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203572.458738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203577.577038:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203577.577353:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203577.577530:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203577.577691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203582.697220:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203582.697226:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203582.697810:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203582.697813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203582.698127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203582.698129:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203582.698452:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203582.698454:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203587.817115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203587.817122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203587.817694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203587.817696:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203587.818001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203587.818004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203587.818321:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203587.818324:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203592.936943:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203592.936948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203592.937335:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203592.937336:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203592.937549:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203592.937568:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203592.937759:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203592.937761:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203598.056885:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203598.056888:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203598.057380:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203598.057382:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203598.057909:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203598.057911:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203598.058138:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203598.058141:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788203603.177036:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203603.177040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203603.177592:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203603.177593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203603.177805:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203603.177807:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203603.177987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203603.177989:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788203608.296933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203608.296937:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203608.297389:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203608.297390:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203608.297594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203608.297596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203608.297786:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203608.297788:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788203613.417143:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203613.417150:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203613.417901:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203613.417904:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203613.418155:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203613.418169:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203613.418411:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203613.418412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788203618.537116:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203618.537124:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203618.537589:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203618.537592:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203618.537859:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203618.537860:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203618.538065:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203618.538067:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203623.657302:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203623.657309:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203623.658190:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203623.658193:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203623.658490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203623.658492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203623.658732:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203623.658734:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203628.777005:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203628.777011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203628.777614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203628.777616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203628.777823:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203628.777837:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203628.778051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203628.778053:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203633.897045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203633.897052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203633.897541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203633.897543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203633.897771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203633.897773:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203633.898022:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203633.898024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203639.017009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203639.017016:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203639.017555:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203639.017557:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203639.017824:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203639.017827:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203639.018079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203639.018082:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203644.137103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203644.137453:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203644.137622:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203644.137788:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203649.257144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203649.257150:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203649.257724:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203649.257727:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203649.258033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203649.258035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203649.258303:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203649.258306:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203654.377138:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203654.377147:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203654.377739:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203654.377742:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203654.378084:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203654.378087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203654.378405:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203654.378407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203659.497032:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203659.497036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203659.497537:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203659.497539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203659.497728:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203659.497730:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203659.497962:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203659.497963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203664.617082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203664.617087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203664.617711:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203664.617713:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203664.618021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203664.618024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203664.618347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203664.618350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203669.737012:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203669.737019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203669.737490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203669.737492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203669.737782:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203669.737797:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203669.738076:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203669.738078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203674.857046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203674.857050:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203674.857705:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203674.857707:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203674.857900:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203674.857901:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203674.858134:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203674.858136:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203679.977014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203679.977021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203679.977442:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203679.977444:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203679.977635:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203679.977637:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203679.977843:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203679.977845:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788203685.097080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203685.097085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203685.097593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203685.097595:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203685.097895:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203685.097911:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788203685.098176:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203685.098178:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203690.217079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203690.217084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203690.217664:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203690.217666:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203690.217838:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203690.217839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203690.218002:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203690.218021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203695.337109:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203695.337115:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203695.337623:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203695.337625:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203695.337866:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203695.337868:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203695.338080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203695.338081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203700.457319:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203700.457328:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203700.457791:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203700.457794:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203700.458063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203700.458066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203700.458322:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203700.458324:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203705.577013:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203705.577019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203705.577533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203705.577535:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203705.577725:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203705.577727:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203705.577884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203705.577900:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203710.697348:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203710.697814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203710.697967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203710.698114:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203715.817200:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203715.817206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203715.817827:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203715.817830:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203715.818107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203715.818110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203715.818424:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203715.818427:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203720.936938:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203720.936941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203720.937431:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203720.937432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203720.937643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203720.937644:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203720.937826:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203720.937828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203726.056850:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203726.056853:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203726.057162:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203726.057163:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203726.057334:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203726.057335:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203726.057507:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203726.057509:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203731.177217:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203731.177223:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203731.177802:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203731.177804:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203731.177995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203731.177997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203731.178237:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203731.178253:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203736.297218:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203736.297225:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203736.297808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203736.297811:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203736.298148:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203736.298151:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203736.298504:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203736.298506:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203741.417056:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203741.417061:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203741.417571:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203741.417573:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203741.417821:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203741.417823:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203741.418075:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203741.418078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203746.536890:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203746.536894:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203746.537234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203746.537236:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203746.537492:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203746.537493:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203746.537681:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203746.537694:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203751.657248:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203751.657254:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203751.658128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203751.658130:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203751.658348:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203751.658350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203751.658614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203751.658616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203756.776988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203756.776993:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203756.777568:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203756.777570:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203756.777808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203756.777810:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203756.778001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203756.778003:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203761.897025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203761.897030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203761.897504:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203761.897505:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203761.897776:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203761.897778:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203761.897957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203761.897970:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203767.017007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203767.017014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203767.017467:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203767.017468:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203767.017652:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203767.017653:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203767.017825:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203767.017826:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203772.137036:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203772.137042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203772.137563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203772.137565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203772.137791:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203772.137793:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203772.138062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203772.138064:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203777.257160:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203777.257527:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203777.257694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203777.257889:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203782.377556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203782.377559:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203782.377997:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203782.378000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203782.378435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203782.378437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203782.378727:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203782.378729:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203787.497571:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203787.497584:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203787.498123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203787.498125:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203787.498324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203787.498326:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203787.498536:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203787.498539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203792.617031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203792.617035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203792.617719:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203792.617722:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203792.617920:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203792.617922:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203792.618150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203792.618152:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203797.737326:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203797.737330:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203797.738207:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203797.738209:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203797.738449:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203797.738451:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203797.738733:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203797.738736:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788203802.856981:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203802.856988:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203802.857637:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203802.857639:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203802.857808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203802.857809:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203802.857978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203802.857980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788203807.977022:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203807.977028:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203807.977533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203807.977536:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203807.977710:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203807.977712:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788203807.977863:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203807.977865:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203813.096946:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203813.096951:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203813.097455:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203813.097457:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203813.097636:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203813.097638:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203813.097819:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203813.097821:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203818.217077:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203818.217083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203818.217475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203818.217477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203818.217747:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203818.217749:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203818.218005:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203818.218021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203823.336976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203823.336980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203823.337436:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203823.337437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203823.337606:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203823.337608:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203823.337816:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203823.337819:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203828.456941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203828.456948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203828.457559:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203828.457561:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203828.457821:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203828.457824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203828.458146:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203828.458148:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203833.577015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203833.577021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203833.577598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203833.577600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203833.577786:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203833.577787:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203833.578040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203833.578041:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203838.696967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203838.696971:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203838.697463:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203838.697464:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203838.697681:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203838.697684:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203838.697880:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203838.697882:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203843.817196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203843.817489:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203843.817642:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203843.817786:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203848.937053:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203848.937072:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203848.937741:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203848.937743:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203848.937994:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203848.937996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203848.938224:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203848.938226:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203854.057090:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203854.057097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203854.057558:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203854.057560:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203854.057741:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203854.057742:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203854.057927:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203854.057928:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203859.176927:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203859.176933:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203859.177409:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203859.177411:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203859.177704:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203859.177707:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203859.177979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203859.177981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203864.296954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203864.296969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203864.297414:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203864.297416:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203864.297597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203864.297599:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203864.297806:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203864.297808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203869.417062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203869.417067:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203869.417631:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203869.417633:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203869.417914:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203869.417917:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203869.418204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203869.418206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203874.536994:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203874.537000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203874.537536:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203874.537539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203874.537824:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203874.537827:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203874.538102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203874.538105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203879.657245:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203879.657271:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203879.657898:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203879.657899:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203879.658144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203879.658146:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203879.658316:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203879.658317:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203884.777063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203884.777069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203884.777721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203884.777723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203884.777938:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203884.777941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203884.778229:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203884.778231:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203889.897123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203889.897131:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203889.897529:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203889.897531:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203889.897718:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203889.897720:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203889.897946:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203889.897948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203895.017337:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203895.017387:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203895.018254:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203895.018256:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203895.018509:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203895.018511:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203895.018789:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203895.018791:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203900.137096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203900.137103:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203900.137668:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203900.137671:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203900.138011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203900.138014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203900.138336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203900.138339:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203905.257039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203905.257046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203905.257614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203905.257616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203905.257854:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203905.257857:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203905.258051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203905.258053:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203910.377299:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203910.377810:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203910.377965:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203910.378129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203915.497150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203915.497158:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203915.497795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203915.497798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203915.498114:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203915.498117:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203915.498472:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203915.498474:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788203920.617141:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203920.617147:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203920.617645:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203920.617648:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203920.617875:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203920.617877:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203920.618089:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203920.618091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788203925.737262:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203925.737277:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203925.738039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203925.738042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203925.738267:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203925.738268:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788203925.738475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203925.738477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203930.857171:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203930.857175:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203930.857636:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203930.857639:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203930.857930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203930.857932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203930.858241:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203930.858243:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203935.977153:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203935.977161:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203935.977820:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203935.977823:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203935.978148:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203935.978151:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203935.978402:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203935.978404:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203941.097285:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203941.097295:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203941.098058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203941.098060:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203941.098253:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203941.098255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203941.098528:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203941.098530:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203946.217128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203946.217134:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203946.217567:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203946.217569:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203946.217813:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203946.217814:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203946.217998:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203946.218000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203951.337266:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203951.337276:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203951.338009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203951.338012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203951.338230:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203951.338232:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203951.338521:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203951.338524:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203956.457250:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203956.457259:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203956.457857:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203956.457860:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203956.458135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203956.458138:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203956.458410:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203956.458412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788203961.577059:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203961.577068:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203961.577485:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203961.577487:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203961.577739:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203961.577741:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203961.577928:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203961.577930:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788203966.697193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203966.697199:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203966.697602:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203966.697605:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203966.697796:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203966.697798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203966.697974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203966.697976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788203971.817035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203971.817042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203971.817552:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203971.817554:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203971.817776:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203971.817778:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203971.817958:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203971.817959:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788203976.937128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203976.937473:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203976.937620:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203976.937778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203982.057556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203982.057559:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788203982.057921:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203982.057923:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788203982.058236:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203982.058239:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788203982.058563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203982.058565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1788203987.177345:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203987.177349:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203987.178097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203987.178099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203987.178290:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203987.178292:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788203987.178494:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788203987.178508:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203992.297111:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203992.297116:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203992.297668:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203992.297670:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203992.297936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203992.297940:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203992.298336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203992.298339:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203997.417305:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203997.417319:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203997.418012:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203997.418014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203997.418253:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203997.418255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203997.418522:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788203997.418526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204002.536953:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204002.536959:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204002.537411:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204002.537412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204002.537629:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204002.537631:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204002.537786:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204002.537788:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204007.656990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204007.656994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204007.657482:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204007.657483:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204007.657649:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204007.657650:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204007.657857:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204007.657859:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204012.776937:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204012.776941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204012.777496:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204012.777499:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204012.777712:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204012.777713:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204012.777941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204012.777953:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204017.897152:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204017.897155:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204017.897404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204017.897406:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204017.897597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204017.897599:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204017.897842:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204017.897844:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204023.016934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204023.016939:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204023.017398:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204023.017400:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204023.017618:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204023.017620:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204023.017779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204023.017780:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204028.136938:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204028.136944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788204028.137299:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204028.137301:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788204028.137541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204028.137543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788204028.137701:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204028.137714:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788204033.256987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204033.256994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788204033.257582:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204033.257585:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788204033.257882:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204033.257884:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788204033.258153:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204033.258155:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788204038.376992:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204038.376997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204038.377587:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204038.377589:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204038.377868:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204038.377870:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204038.378119:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204038.378122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204043.497090:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204043.497556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204043.497730:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204043.497881:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204048.617252:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204048.617261:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204048.617834:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204048.617836:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204048.618185:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204048.618188:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204048.618559:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204048.618562:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788204053.736797:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204053.736801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788204053.737144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204053.737157:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788204053.737324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204053.737326:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788204053.737495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204053.737496:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788204058.857077:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204058.857081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788204058.857490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204058.857492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788204058.857684:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204058.857686:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788204058.857862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204058.857864:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204063.977135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204063.977142:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204063.978001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204063.978003:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204063.978217:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204063.978219:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204063.978447:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204063.978448:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204069.097014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204069.097021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204069.097475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204069.097478:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204069.097712:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204069.097714:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204069.097876:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204069.097888:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204074.217169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204074.217174:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204074.217619:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204074.217621:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204074.217796:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204074.217798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204074.217954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204074.217955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204079.337012:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204079.337015:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204079.337547:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204079.337549:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204079.337757:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204079.337758:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204079.337917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204079.337919:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204084.456919:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204084.456924:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204084.457472:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204084.457473:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204084.457664:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204084.457666:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204084.457816:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204084.457828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788204089.577165:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204089.577172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204089.577699:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204089.577701:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204089.577985:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204089.577987:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204089.578256:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204089.578258:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204094.697129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204094.697133:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204094.697698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204094.697714:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204094.697945:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204094.697947:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204094.698133:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204094.698135:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204099.817074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204099.817079:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204099.817588:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204099.817591:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204099.817796:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204099.817798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204099.818009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204099.818011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204104.936963:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204104.936967:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788204104.937522:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204104.937524:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788204104.937685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204104.937686:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788204104.937835:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204104.937836:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788204110.057045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204110.057460:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204110.057674:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204110.057814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204115.177635:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204115.177639:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788204115.178080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204115.178083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788204115.178487:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204115.178490:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788204115.178849:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204115.178853:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204120.297085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204120.297094:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204120.297569:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204120.297571:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204120.297851:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204120.297853:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204120.298105:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204120.298107:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204125.417031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204125.417038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204125.417581:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204125.417588:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204125.417879:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204125.417883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204125.418206:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204125.418209:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788204130.537130:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204130.537138:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204130.537778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204130.537780:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204130.538046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204130.538048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204130.538328:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204130.538331:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204135.657049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204135.657055:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204135.657475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204135.657476:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204135.657719:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204135.657721:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204135.657947:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204135.657949:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204140.777043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204140.777048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204140.777690:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204140.777692:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204140.777955:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204140.777958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204140.778261:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204140.778276:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204145.897192:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204145.897198:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204145.897917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204145.897920:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204145.898245:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204145.898248:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204145.898603:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204145.898606:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788204151.017050:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204151.017069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788204151.017637:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204151.017640:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788204151.017860:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204151.017862:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788204151.018110:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204151.018112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788204156.137014:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204156.137021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204156.137457:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204156.137459:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204156.137688:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204156.137690:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204156.137849:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204156.137851:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204161.257185:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204161.257189:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204161.257917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204161.257918:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204161.258188:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204161.258190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204161.258471:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204161.258473:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204166.376928:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204166.376932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204166.377462:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204166.377464:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204166.377653:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204166.377654:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204166.377882:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204166.377884:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204171.496974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204171.496978:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204171.497358:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204171.497377:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204171.497578:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204171.497579:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204171.497762:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204171.497774:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204176.617003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204176.617240:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204176.617335:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204176.617498:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204181.737217:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204181.737223:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204181.737872:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204181.737876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204181.738198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204181.738201:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204181.738509:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204181.738511:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204186.857132:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204186.857139:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204186.857763:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204186.857766:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204186.858009:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204186.858011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204186.858267:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204186.858268:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204191.977150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204191.977155:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204191.977765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204191.977767:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204191.978068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204191.978071:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204191.978342:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204191.978344:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204197.096956:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204197.096963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204197.097323:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204197.097325:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204197.097565:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204197.097566:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204197.097779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204197.097781:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204202.217107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204202.217111:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204202.217685:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204202.217688:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204202.217930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204202.217932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204202.218194:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204202.218196:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204207.337103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204207.337110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204207.337612:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204207.337613:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204207.337890:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204207.337892:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204207.338118:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204207.338120:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204212.456940:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204212.456945:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204212.457595:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204212.457597:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204212.457796:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204212.457797:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204212.458044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204212.458046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204217.577054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204217.577061:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204217.577705:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204217.577708:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204217.577951:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204217.577953:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204217.578211:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204217.578214:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204222.696919:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204222.696925:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204222.697402:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204222.697404:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204222.697598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204222.697600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204222.697806:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204222.697808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788204227.817237:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204227.817255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204227.818155:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204227.818157:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204227.818339:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204227.818341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204227.818694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204227.818695:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204232.936881:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204232.936885:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204232.937267:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204232.937270:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204232.937556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204232.937559:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204232.937829:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204232.937832:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204238.057008:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204238.057015:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204238.057538:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204238.057540:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204238.057739:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204238.057741:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204238.057936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204238.057938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788204243.177110:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204243.177433:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204243.177564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204243.177714:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204248.297454:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204248.297457:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204248.297797:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204248.297799:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204248.298082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204248.298084:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204248.298360:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204248.298386:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204253.417272:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204253.417281:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204253.417924:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204253.417927:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204253.418205:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204253.418207:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204253.418517:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204253.418520:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204258.537097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204258.537103:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204258.537682:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204258.537685:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204258.537989:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204258.537991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204258.538257:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204258.538259:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788204263.657021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204263.657027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204263.657628:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204263.657631:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204263.657902:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204263.657904:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204263.658147:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204263.658149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00010000:00000001:1.0:1788204265.896475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204265.896476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204265.896478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204265.896479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204265.896481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204265.896482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204266.664469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204266.664477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204266.664479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204266.664484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204266.664488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204266.664492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204266.664494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204266.664497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204266.664500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204266.728531:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204266.728545:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204266.728550:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204266.728566:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204266.728577:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204266.728587:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204266.728621:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204266.728626:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204266.728630:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204266.920520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204266.920526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204266.920532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204266.920533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204266.920536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204266.920537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204266.920540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204266.920541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204266.920556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204266.920558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204266.920561:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204267.688463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204267.688470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204267.688472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204267.688477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204267.688481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204267.688485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204267.688488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204267.688490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204267.688493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204267.752442:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204267.752454:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204267.752460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204267.752470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204267.752477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204267.752484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204267.752502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204267.752506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204267.752509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204267.944494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204267.944502:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204267.944505:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204267.944508:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944510:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204267.944512:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944514:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944516:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204267.944521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204267.944525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204267.944527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204268.264493:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300395009, at 4300395520 00000100:00000010:1.0:1788204268.264517:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9300. 00000100:00000001:1.0:1788204268.712493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204268.712504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204268.712507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204268.712514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204268.712519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204268.712524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204268.712527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204268.712531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204268.712535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204268.776496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204268.776505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204268.776508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204268.776517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204268.776524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204268.776531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204268.776536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204268.776538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204268.776542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204268.777129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204268.777133:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204268.777755:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204268.777756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204268.778003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204268.778004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204268.778253:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204268.778255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00010000:00000001:1.0:1788204268.968551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204268.968562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204268.968568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204268.968570:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204268.968573:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204268.968575:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204268.968578:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204268.968580:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204268.968607:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204268.968610:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204268.968613:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204269.736491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204269.736500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204269.736502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204269.736506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204269.736509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204269.736513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204269.736515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204269.736518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204269.736520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204269.800491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204269.800502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204269.800507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204269.800549:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204269.800553:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204269.800555:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204269.800558:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204269.800561:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204269.800563:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204269.992440:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204269.992448:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204269.992452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204269.992453:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204269.992456:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204269.992457:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204269.992460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204269.992460:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204270.760452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204270.760462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204270.760464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204270.760470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204270.760473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204270.760477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204270.760479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204270.760481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204270.760483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204270.824462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204270.824471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204270.824474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204270.824482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204270.824488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204270.824499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204270.824510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204270.824514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204270.824517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204271.016541:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204271.016544:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204271.016547:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016551:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204271.016553:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016554:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016556:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204271.016560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204271.016564:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204271.016565:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204271.016566:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016567:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204271.016568:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016569:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016570:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016573:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204271.016573:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204271.016574:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204271.016575:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016576:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204271.016577:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016577:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204271.016579:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204271.016594:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204271.016596:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204271.016597:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204271.784525:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204271.784536:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204271.784538:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204271.784545:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204271.784549:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204271.784554:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204271.784556:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204271.784558:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204271.784561:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204271.848503:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204271.848513:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204271.848517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204271.848525:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204271.848531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204271.848535:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204271.848544:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204271.848547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204271.848550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204272.040512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204272.040523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204272.040530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204272.040531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204272.040535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204272.040537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204272.040540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204272.040542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204272.808459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204272.808467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204272.808469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204272.808475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204272.808478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204272.808481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204272.808484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204272.808486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204272.808489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204272.872474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204272.872481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204272.872485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204272.872492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204272.872497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204272.872502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204272.872505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204272.872509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204272.872512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204273.064487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204273.064492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204273.064495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204273.064496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204273.064498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204273.064499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204273.064501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204273.064501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204273.064514:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204273.064515:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204273.064517:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204273.384469:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300400129, at 4300400640 00000100:00000010:1.0:1788204273.384484:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000001:1.0:1788204273.832521:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204273.832531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204273.832533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204273.832539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204273.832551:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204273.832558:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204273.832562:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204273.832564:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204273.832568:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204273.896482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204273.896493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204273.896498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204273.896506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204273.896515:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204273.896520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204273.896536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204273.896540:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204273.896543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204273.897137:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204273.897142:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204273.897915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204273.897918:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204273.898225:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204273.898228:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204273.898540:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204273.898543:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00010000:00000001:1.0:1788204274.088630:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204274.088640:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204274.088650:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204274.088650:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204274.088653:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204274.088654:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204274.088657:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204274.088658:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204274.856473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204274.856482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204274.856485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204274.856491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204274.856496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204274.856500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204274.856503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204274.856506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204274.856510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204274.920492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204274.920500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204274.920502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204274.920511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204274.920516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204274.920521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204274.920531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204274.920533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204274.920536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204275.112542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204275.112548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204275.112553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204275.112554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204275.112556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204275.112557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204275.112560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204275.112560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204275.112585:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204275.112587:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204275.112589:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204275.880507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204275.880517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204275.880519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204275.880526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204275.880530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204275.880535:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204275.880539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204275.880541:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204275.880544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204275.944436:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204275.944445:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204275.944448:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204275.944454:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204275.944459:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204275.944464:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204275.944466:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204275.944470:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204275.944473:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204276.136529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204276.136539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204276.136547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204276.136550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204276.136553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204276.136554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204276.136559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204276.136560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204276.904521:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204276.904531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204276.904533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204276.904540:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204276.904544:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204276.904549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204276.904552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204276.904554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204276.904557:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204276.968487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204276.968495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204276.968498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204276.968503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204276.968506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204276.968510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204276.968518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204276.968521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204276.968526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204277.160538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204277.160545:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204277.160548:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204277.160551:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160553:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204277.160555:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160557:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160558:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204277.160562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204277.160564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204277.160564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204277.160592:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204277.160595:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204277.160597:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204277.928545:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204277.928555:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204277.928558:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204277.928565:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204277.928569:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204277.928574:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204277.928577:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204277.928580:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204277.928583:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204277.992506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204277.992516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204277.992519:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204277.992526:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204277.992531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204277.992537:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204277.992540:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204277.992544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204277.992554:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204278.184506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204278.184513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204278.184517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204278.184518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204278.184522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204278.184524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204278.184527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204278.184529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204278.504482:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300405250, at 4300405760 00000100:00000010:1.0:1788204278.504505:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000001:1.0:1788204278.952486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204278.952493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204278.952494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204278.952499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204278.952502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204278.952506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204278.952509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204278.952512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204278.952515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204279.016463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204279.016471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204279.016475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204279.016481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204279.016487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204279.016491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204279.016500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204279.016503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204279.016506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204279.017028:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204279.017032:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204279.017359:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204279.017361:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204279.017616:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204279.017618:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204279.017808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204279.017809:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00010000:00000001:1.0:1788204279.208491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204279.208498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204279.208502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204279.208503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204279.208506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204279.208507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204279.208509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204279.208509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204279.208522:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204279.208523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204279.208525:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204279.976498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204279.976506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204279.976508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204279.976514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204279.976517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204279.976521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204279.976523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204279.976526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204279.976528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204280.040483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204280.040492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204280.040496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204280.040505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204280.040510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204280.040516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204280.040521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204280.040526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204280.040529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204280.232477:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204280.232479:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204280.232481:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232486:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204280.232487:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232489:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232491:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204280.232497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204280.232503:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204280.232504:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204280.232505:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232506:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204280.232506:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232507:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232508:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204280.232512:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204280.232513:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204280.232514:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232516:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204280.232517:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232518:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204280.232520:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204281.000460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204281.000465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204281.000467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204281.000470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204281.000473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204281.000476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204281.000477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204281.000479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204281.000481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204281.064479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204281.064484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204281.064487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204281.064493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204281.064499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204281.064505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204281.064509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204281.064511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204281.064514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204281.256553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204281.256558:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204281.256561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204281.256562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204281.256564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204281.256565:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204281.256567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204281.256568:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204281.256583:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204281.256585:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204281.256586:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204282.024485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204282.024492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204282.024494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204282.024498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204282.024501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204282.024504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204282.024506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204282.024507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204282.024509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204282.088466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204282.088474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204282.088478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204282.088486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204282.088492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204282.088496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204282.088499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204282.088501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204282.088505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204282.280469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204282.280476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204282.280482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204282.280484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204282.280487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204282.280488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204282.280492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204282.280493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204283.048446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204283.048451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204283.048452:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204283.048456:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204283.048458:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204283.048461:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204283.048463:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204283.048464:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204283.048466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204283.112453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204283.112460:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204283.112464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204283.112471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204283.112477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204283.112482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204283.112493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204283.112495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204283.112503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204283.304505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204283.304511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204283.304517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204283.304518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204283.304521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204283.304522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204283.304525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204283.304526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204283.304548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204283.304550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204283.304552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204283.624468:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300410369, at 4300410880 00000100:00000010:1.0:1788204283.624483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000001:1.0:1788204284.072479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204284.072484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204284.072486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204284.072490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204284.072493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204284.072496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204284.072497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204284.072499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204284.072501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204284.136477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204284.136485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204284.136488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204284.136497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204284.136503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204284.136508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204284.136511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204284.136515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204284.136518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204284.137119:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204284.137124:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204284.137704:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204284.137707:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204284.137933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204284.137936:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204284.138101:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204284.138102:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204284.328461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204284.328467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204284.328473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204284.328475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204284.328478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204284.328479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204284.328484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204284.328485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204285.096451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204285.096458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204285.096460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204285.096465:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204285.096468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204285.096472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204285.096475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204285.096477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204285.096480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204285.160431:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204285.160438:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204285.160441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204285.160448:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204285.160459:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204285.160466:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204285.160478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204285.160481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204285.160484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204285.352513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204285.352518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204285.352522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204285.352523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204285.352524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204285.352525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204285.352528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204285.352528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204285.352541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204285.352543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204285.352545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204286.120462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204286.120468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204286.120469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204286.120473:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204286.120475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204286.120478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204286.120480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204286.120481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204286.120483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204286.184477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204286.184486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204286.184489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204286.184496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204286.184502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204286.184506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204286.184518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204286.184521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204286.184524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204286.376465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204286.376470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204286.376474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204286.376475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204286.376477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204286.376478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204286.376480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204286.376481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204287.144517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204287.144527:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204287.144529:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204287.144536:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204287.144548:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204287.144553:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204287.144556:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204287.144559:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204287.144563:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204287.208490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204287.208501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204287.208503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204287.208511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204287.208517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204287.208523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204287.208526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204287.208530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204287.208533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204287.400507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204287.400514:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204287.400517:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204287.400520:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400522:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204287.400525:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400526:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400528:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204287.400531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204287.400533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204287.400534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204287.400549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204287.400552:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204287.400554:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204288.168479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204288.168489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204288.168492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204288.168499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204288.168503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204288.168508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204288.168511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204288.168514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204288.168517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204288.232513:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204288.232522:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204288.232525:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204288.232534:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204288.232540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204288.232546:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204288.232550:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204288.232552:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204288.232557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204288.424487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204288.424496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204288.424503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204288.424504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204288.424508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204288.424509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204288.424513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204288.424515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204288.744453:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300415489, at 4300416000 00000100:00000010:1.0:1788204288.744470:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:1.0:1788204289.192506:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204289.192515:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204289.192518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204289.192524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204289.192529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204289.192534:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204289.192536:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204289.192540:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204289.192543:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204289.256483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204289.256492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204289.256495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204289.256501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204289.256507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204289.256514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204289.256519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204289.256526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204289.256530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204289.257066:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204289.257074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204289.257495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204289.257498:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204289.257788:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204289.257791:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204289.258085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204289.258087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204289.448502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204289.448508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204289.448513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204289.448514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204289.448516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204289.448517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204289.448519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204289.448520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204289.448542:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204289.448544:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204289.448546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204290.216465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204290.216472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204290.216473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204290.216478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204290.216482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204290.216485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204290.216488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204290.216490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204290.216493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204290.280481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204290.280489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204290.280494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204290.280501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204290.280508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204290.280515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204290.280529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204290.280532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204290.280535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204290.472487:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204290.472490:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204290.472494:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472499:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204290.472501:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472506:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204290.472511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204290.472515:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204290.472516:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204290.472518:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472519:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204290.472520:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472522:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472524:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204290.472528:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204290.472530:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204290.472531:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472532:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204290.472533:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472535:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204290.472537:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204291.240495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204291.240503:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204291.240505:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204291.240511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204291.240514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204291.240518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204291.240521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204291.240523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204291.240526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204291.304459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204291.304469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204291.304473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204291.304482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204291.304488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204291.304494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204291.304499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204291.304502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204291.304507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204291.496496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204291.496501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204291.496504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204291.496505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204291.496514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204291.496514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204291.496517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204291.496518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204291.496530:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204291.496532:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204291.496534:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204292.264432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204292.264438:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204292.264439:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204292.264443:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204292.264445:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204292.264448:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204292.264450:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204292.264451:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204292.264459:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204292.328487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204292.328496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204292.328500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204292.328507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204292.328513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204292.328519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204292.328522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204292.328525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204292.328533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204292.520462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204292.520468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204292.520474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204292.520475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204292.520478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204292.520479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204292.520482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204292.520484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204293.288520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204293.288530:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204293.288532:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204293.288539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204293.288543:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204293.288547:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204293.288550:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204293.288553:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204293.288556:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204293.352436:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204293.352444:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204293.352446:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204293.352453:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204293.352460:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204293.352467:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204293.352480:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204293.352484:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204293.352486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204293.544581:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204293.544590:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204293.544597:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204293.544598:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204293.544601:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204293.544602:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204293.544606:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204293.544607:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204293.544633:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204293.544636:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204293.544640:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204293.864470:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300420609, at 4300421120 00000100:00000010:1.0:1788204293.864492:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000001:1.0:1788204294.312424:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204294.312432:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204294.312434:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204294.312438:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204294.312441:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204294.312445:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204294.312447:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204294.312449:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204294.312452:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204294.376477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204294.376486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204294.376489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204294.376497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204294.376503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204294.376507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204294.376511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204294.376514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204294.376521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788204294.377067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204294.377073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204294.377549:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204294.377551:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204294.377761:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204294.377762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788204294.377941:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204294.377944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000020:00000040:0.0:1788204294.378120:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204294.378122:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204294.378124:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788204294.378126:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204294.378127:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204294.378128:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788204294.378130:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204294.378132:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204294.378134:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204294.378137:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204294.378138:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204294.378140:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204294.378142:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204294.568488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204294.568496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204294.568502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204294.568503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204294.568505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204294.568506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204294.568509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204294.568510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204295.336503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204295.336513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204295.336515:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204295.336522:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204295.336526:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204295.336532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204295.336535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204295.336538:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204295.336542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204295.400485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204295.400497:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204295.400502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204295.400510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204295.400517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204295.400524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204295.400539:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204295.400543:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204295.400547:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204295.592557:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204295.592567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204295.592588:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204295.592589:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204295.592593:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204295.592594:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204295.592598:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204295.592599:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204295.592622:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204295.592626:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204295.592630:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204296.360499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204296.360509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204296.360512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204296.360519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204296.360523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204296.360529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204296.360531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204296.360534:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204296.360536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204296.424499:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204296.424507:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204296.424510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204296.424518:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204296.424524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204296.424528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204296.424532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204296.424536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204296.424539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204296.616441:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204296.616446:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204296.616452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204296.616454:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204296.616457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204296.616458:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204296.616460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204296.616462:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204296.616518:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204296.616522:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204296.616526:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204297.384481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204297.384487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204297.384488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204297.384492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204297.384495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204297.384497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204297.384500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204297.384501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204297.384503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204297.448468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204297.448474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204297.448477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204297.448483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204297.448487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204297.448491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204297.448495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204297.448498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204297.448501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204297.640514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204297.640522:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204297.640524:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204297.640526:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640528:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204297.640530:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640531:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640533:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204297.640536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204297.640537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204297.640538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204297.640551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204297.640553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204297.640555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204298.408451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204298.408457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204298.408459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204298.408462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204298.408464:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204298.408468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204298.408469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204298.408471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204298.408473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204298.472463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204298.472470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204298.472474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204298.472481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204298.472488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204298.472492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204298.472500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204298.472502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204298.472504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204298.664468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204298.664472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204298.664476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204298.664477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204298.664479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204298.664480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204298.664482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204298.664483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204298.664532:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204298.664536:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204298.664539:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204298.984467:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300425729, at 4300426240 00000100:00000010:1.0:1788204298.984479:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9360. 00000100:00000040:0.0:1788204298.984586:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300425729, at 4300426240 00000100:00000010:0.0:1788204298.984593:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000001:1.0:1788204299.432460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204299.432466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204299.432467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204299.432471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204299.432473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204299.432476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204299.432478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204299.432481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204299.432483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204299.496453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204299.496458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204299.496462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204299.496467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204299.496473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204299.496477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204299.496481:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204299.496484:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204299.496488:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.496703:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204299.496708:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204299.496712:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204299.496719:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.496730:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204299.496733:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.496736:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204299.496740:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204299.496745:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204299.496749:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205848704:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204299.496755:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204299.496759:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204299.496762:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.496764:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204299.496770:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204299.496774:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204299.496782:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204299.496788:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da058. 00000400:00000010:0.0:1788204299.496804:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192172a8. 00000100:00000200:0.0:1788204299.496812:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205848704, portal 25 00000100:00000001:0.0:1788204299.496816:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204299.496819:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000040:0.0:1788204299.496828:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.496839:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204299.496842:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204299.496846:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788204299.496849:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205848704, offset 0 00000400:00000200:0.0:1788204299.496856:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204299.496865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.496872:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204299.496879:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204299.496884:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204299.496889:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204299.496894:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204299.496899:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58486:58486:256:4294967295] 192.168.201.151@tcp LPNI seq info [58466:58466:8:4294967295] 00000400:00000200:0.0:1788204299.496906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.496917:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204299.496923:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.496927:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788204299.496936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204299.496941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204299.496948:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204299.496959:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788204299.496966:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.496969:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:0.0:1788204299.496976:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204299.496987:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204299.496999:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:0.0:1788204299.497004:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:0.0:1788204299.497012:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204299.497020:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497034:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497039:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497048:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204299.497055:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204299.497061:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204299.497065:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300430849 00000100:00000010:0.0:1788204299.497071:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204299.497078:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204299.497084:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300430849 00000100:00000010:0.0:1788204299.497088:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff90c0. 00000100:00000001:0.0:1788204299.497093:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204299.497095:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204299.497096:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497098:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204299.497103:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204299/real 1788204299] req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204299.497115:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204299.497117:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204299.497121:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da058. 00000400:00000200:0.0:1788204299.497124:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192172a8 00000400:00000010:0.0:1788204299.497127:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192172a8. 00000100:00000001:0.0:1788204299.497131:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204299.497136:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204299.497156:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497167:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204299.497169:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497173:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497174:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204299.497180:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497189:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204299.497195:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497209:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204299.497213:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497216:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497218:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497229:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204299.497232:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204299.497236:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204299.497242:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204299.497246:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204299.497250:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204299.497259:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205848704:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204299.497266:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497270:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497274:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205848704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497298:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204299.497300:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497303:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204299.497307:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204299.497311:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204299.497313:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788204299.497316:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204299.497320:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:0.0:1788204299.497324:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204299.497326:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204299.497328:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:0.0:1788204299.497333:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204299.497335:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497339:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497344:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204299.497347:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497351:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204299.497353:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204299.497356:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204299.497362:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497408:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204299.497411:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497421:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204299.497425:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204299.497427:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204299.497431:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205848832:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204299.497434:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204299.497435:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204299.497437:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497438:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497441:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204299.497443:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204299.497446:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204299.497449:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da058. 00000400:00000010:0.0:1788204299.497452:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192172a8. 00000100:00000200:0.0:1788204299.497454:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205848832, portal 10 00000100:00000001:0.0:1788204299.497456:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204299.497457:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:0.0:1788204299.497464:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497469:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204299.497471:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204299.497472:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788204299.497474:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205848832, offset 0 00000400:00000200:0.0:1788204299.497477:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497481:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497484:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204299.497487:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204299.497489:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204299.497491:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204299.497493:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497496:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58487:58487:256:4294967295] 192.168.201.151@tcp LPNI seq info [58467:58467:8:4294967295] 00000400:00000200:0.0:1788204299.497499:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497504:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204299.497508:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497510:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788204299.497514:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204299.497516:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204299.497519:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204299.497522:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788204299.497525:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497526:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:0.0:1788204299.497530:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204299.497532:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204299.497535:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:0.0:1788204299.497537:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:0.0:1788204299.497539:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204299.497542:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497549:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497551:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497556:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497557:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204299.497560:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204299.497563:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204299.497566:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204299.497568:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300426907 00000100:00000001:0.0:1788204299.497570:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497572:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300426907 00000100:00000001:0.0:1788204299.497574:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497576:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204299.497578:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497579:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204299.497583:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204299/real 1788204299] req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204299.497589:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204299.497590:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204299.497592:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da058. 00000400:00000200:0.0:1788204299.497594:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192172a8 00000400:00000010:0.0:1788204299.497595:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192172a8. 00000100:00000001:0.0:1788204299.497597:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204299.497599:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204299.497606:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497618:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204299.497619:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497621:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497621:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204299.497624:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497628:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204299.497631:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497634:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204299.497636:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497637:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497637:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497638:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204299.497640:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204299.497641:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204299.497643:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204299.497644:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204299.497646:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204299.497650:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205848832:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204299.497652:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204299.497654:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497671:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204299.497672:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497674:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204299.497675:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204299.497676:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204299.497678:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205848960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204299.497680:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204299.497681:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204299.497682:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497683:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497686:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788204299.497687:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204299.497689:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204299.497691:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da058. 00000400:00000010:0.0:1788204299.497693:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192172a8. 00000100:00000200:0.0:1788204299.497695:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205848960, portal 4 00000100:00000001:0.0:1788204299.497696:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204299.497697:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:0.0:1788204299.497701:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497704:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204299.497706:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204299.497707:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788204299.497708:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205848960, offset 0 00000400:00000200:0.0:1788204299.497710:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497713:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497715:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204299.497716:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204299.497718:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204299.497719:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204299.497721:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497723:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58488:58488:256:4294967295] 192.168.201.151@tcp LPNI seq info [58468:58468:8:4294967295] 00000400:00000200:0.0:1788204299.497725:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497729:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204299.497732:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497734:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788204299.497736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204299.497738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204299.497740:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204299.497742:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788204299.497755:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497757:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:0.0:1788204299.497759:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204299.497761:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204299.497763:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:0.0:1788204299.497764:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:0.0:1788204299.497766:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204299.497768:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497772:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497774:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497778:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497779:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204299.497781:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204299.497783:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204299.497784:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204299.497786:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300426793 00000100:00000001:0.0:1788204299.497787:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497789:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300426793 00000100:00000001:0.0:1788204299.497790:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497791:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204299.497792:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497793:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204299.497795:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204299/real 1788204299] req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204299.497799:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204299.497801:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204299.497802:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da058. 00000400:00000200:0.0:1788204299.497803:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192172a8 00000400:00000010:0.0:1788204299.497804:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192172a8. 00000100:00000001:0.0:1788204299.497805:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204299.497806:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204299.497810:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497814:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204299.497815:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497816:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497817:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204299.497819:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497822:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204299.497824:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497827:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204299.497829:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497829:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.497830:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497831:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204299.497832:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204299.497834:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204299.497835:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204299.497836:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204299.497839:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204299.497843:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205848960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204299.497845:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497846:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497848:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205848832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497851:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204299.497852:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497853:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204299.497854:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204299.497855:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204299.497856:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788204299.497857:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204299.497858:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:0.0:1788204299.497859:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204299.497860:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204299.497860:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:0.0:1788204299.497862:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204299.497863:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497864:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497865:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205848960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497869:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204299.497870:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497870:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788204299.497871:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204299.497872:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204299.497873:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788204299.497874:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204299.497874:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:0.0:1788204299.497875:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204299.497876:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204299.497877:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:0.0:1788204299.497878:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204299.497879:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497880:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.497882:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204299.497883:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497884:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204299.497885:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204299.497886:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204299.497888:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497891:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204299.497892:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497894:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204299.497895:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204299.497896:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204299.497897:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205849088:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204299.497899:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204299.497900:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204299.497901:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204299.497902:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204299.497904:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788204299.497905:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204299.497907:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204299.497908:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da058. 00000400:00000010:0.0:1788204299.497909:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192172a8. 00000100:00000200:0.0:1788204299.497911:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205849088, portal 4 00000100:00000001:0.0:1788204299.497912:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204299.497913:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:0.0:1788204299.497916:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497920:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204299.497921:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204299.497922:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788204299.497924:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205849088, offset 0 00000400:00000200:0.0:1788204299.497926:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497928:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497930:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204299.497931:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204299.497933:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204299.497935:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204299.497936:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204299.497938:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58489:58489:256:4294967295] 192.168.201.151@tcp LPNI seq info [58469:58469:8:4294967295] 00000400:00000200:0.0:1788204299.497940:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204299.497943:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204299.497946:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497947:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788204299.497949:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204299.497951:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788204299.497953:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204299.497955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788204299.497957:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204299.497958:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:0.0:1788204299.497959:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204299.497961:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204299.497963:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:0.0:1788204299.497964:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:0.0:1788204299.497965:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204299.497967:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497971:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.497973:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204324 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.497978:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.497979:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204299.497982:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204299.497984:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204299.497987:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204299.497989:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300426799 00000100:00000001:0.0:1788204299.497991:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497992:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300426799 00000100:00000001:0.0:1788204299.497994:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300430849 00000100:00000001:0.0:1788204299.497996:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204299.497998:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.498000:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204299.498004:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204299/real 1788204299] req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204299.498012:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204299.498013:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204299.498015:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da058. 00000400:00000200:0.0:1788204299.498017:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192172a8 00000400:00000010:0.0:1788204299.498019:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192172a8. 00000100:00000001:0.0:1788204299.498021:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204299.498024:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204299.498029:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.498033:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204299.498034:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498035:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.498036:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204299.498038:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.498042:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204299.498044:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.498049:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204299.498050:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498051:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204299.498052:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498053:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204299.498054:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204299.498055:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204299.498057:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204299.498058:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204299.498060:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204299.498064:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205849088:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204299.498066:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498067:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204299.498069:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205849088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204324 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204299.498072:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204299.498073:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204299.498074:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788204299.498075:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204299.498076:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204299.498077:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204299.498078:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204299.498079:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:0.0:1788204299.498080:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204299.498081:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204299.498082:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:0.0:1788204299.498083:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204299.498084:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498085:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204299.498087:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204299.498088:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204299.498089:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204299.498090:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204299.688494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204299.688499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204299.688502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204299.688503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204299.688506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204299.688506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204299.688509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204299.688509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204299.688521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204299.688523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204299.688524:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204300.456455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204300.456461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204300.456463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204300.456468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204300.456471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204300.456475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204300.456477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204300.456479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204300.456482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204300.520457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204300.520465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204300.520469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204300.520474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204300.520479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204300.520485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204300.520489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204300.520493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204300.520495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204300.712477:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204300.712479:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204300.712481:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712485:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204300.712486:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712488:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712490:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204300.712493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204300.712496:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204300.712497:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204300.712498:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712499:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204300.712500:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712501:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712502:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204300.712505:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204300.712505:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204300.712506:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712507:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204300.712508:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712509:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204300.712510:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204300.712558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204300.712562:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204300.712564:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204301.480504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204301.480512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204301.480514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204301.480519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204301.480522:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204301.480526:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204301.480528:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204301.480530:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204301.480533:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204301.544467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204301.544475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204301.544478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204301.544483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204301.544489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204301.544495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204301.544501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204301.544506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204301.544511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204301.736497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204301.736504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204301.736510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204301.736511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204301.736514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204301.736515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204301.736519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204301.736520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204301.736534:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204301.736536:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204301.736538:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204302.504487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204302.504494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204302.504496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204302.504501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204302.504504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204302.504507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204302.504510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204302.504513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204302.504516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204302.568479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204302.568483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204302.568487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204302.568492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204302.568498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204302.568502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204302.568512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204302.568515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204302.568517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204302.760478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204302.760485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204302.760490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204302.760492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204302.760495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204302.760496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204302.760499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204302.760500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204302.760551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204302.760554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204302.760556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204303.528470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204303.528476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204303.528478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204303.528482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204303.528485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204303.528488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204303.528490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204303.528491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204303.528493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204303.592464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204303.592472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204303.592476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204303.592483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204303.592489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204303.592494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204303.592498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204303.592502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204303.592505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204303.784517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204303.784523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204303.784529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204303.784530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204303.784533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204303.784535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204303.784538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204303.784539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204303.784558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204303.784560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204303.784563:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204304.104445:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300430849, at 4300431360 00000100:00000040:1.0:1788204304.104446:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300430849, at 4300431360 00000100:00000010:0.0:1788204304.104460:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000010:1.0:1788204304.104472:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff90c0. 00000100:00000001:1.0:1788204304.552437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204304.552442:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204304.552444:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204304.552448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204304.552450:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204304.552453:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204304.552455:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204304.552456:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204304.552458:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204304.616452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204304.616459:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204304.616461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204304.616466:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204304.616467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204304.616473:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204304.616474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204304.616476:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204304.616479:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204304.616479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204304.616485:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204304.616485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204304.616489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204304.616493:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5759 00000100:00000001:1.0:1788204304.616495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204304.616502:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204304.616507:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204304.616510:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204304.616513:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616515:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616517:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616519:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204304.616521:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204304.616522:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204304.616524:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204304.616526:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204304.616528:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204304.616529:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204304.616531:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616533:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616534:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616535:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204304.616538:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204304.616540:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616541:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204304.616543:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204304.616545:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204304.616547:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616548:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204304.616549:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204304.616556:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204304.616558:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204304.616561:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204304.616562:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616564:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204304.616565:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204304.616591:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204304.616598:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204304.616604:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204304.616611:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205849216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204304.616617:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204304.616620:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204304.616623:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205849216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204304.616641:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204304.616642:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204304.616648:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204304.616649:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204304.616650:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204304.616652:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204304.616654:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5759 00000400:00000200:0.0:1788204304.616656:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204304.616659:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204304.616660:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204304.616663:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616664:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616666:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616668:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204304.616670:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204304.616670:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204304.616671:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204304.616673:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204304.616674:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204304.616675:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204304.616676:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616678:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616679:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616683:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204304.616687:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204304.616693:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616697:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204304.616700:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204304.616707:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204304.616712:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616717:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204304.616728:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204304.616734:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616739:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204304.616750:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204304.616756:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204304.616759:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204304.616762:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616766:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204304.616768:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204304.616772:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204304.616778:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204304.616782:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204304.616792:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205849344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204304.616806:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204304.616811:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204304.616821:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205849344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204304.616838:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204304.616841:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204304.616845:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204304.616848:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204304.616850:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204304.616853:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204304.616858:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5759 00000400:00000200:0.0:1788204304.616865:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204304.616872:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204304.616876:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204304.616879:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616882:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616887:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616892:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204304.616896:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204304.616899:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204304.616904:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204304.616908:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204304.616911:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204304.616913:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204304.616918:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616922:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.616926:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616930:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204304.616936:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204304.616941:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204304.616947:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204304.616951:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204304.616956:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204304.616962:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616966:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204304.616972:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.1:1788204304.616972:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204304.616976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000008:00000001:0.0:1788204304.616978:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.616983:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204304.616990:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204304.616997:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204304.617003:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204304.617006:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.617010:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204304.617013:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204304.617017:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204304.617023:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204304.617030:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204304.617041:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205849472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204304.617059:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204304.617063:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204304.617073:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205849472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204304.617085:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204304.617089:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204304.617096:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204304.617101:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204304.617104:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204304.617111:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204304.617117:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5759 00000400:00000200:0.0:1788204304.617127:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204304.617131:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204304.617134:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204304.617139:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.617143:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.617149:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.617154:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204304.617159:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204304.617162:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204304.617166:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204304.617169:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204304.617174:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204304.617180:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204304.617187:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.617192:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204304.617199:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204304.617205:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204304.617212:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204304.617220:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204304.617226:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204304.617230:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204304.617234:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204304.617240:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.617243:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204304.617248:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204304.617254:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.617257:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204304.617262:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204304.617268:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204304.617270:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204304.617272:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204304.617273:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204304.617274:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204304.617276:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204304.617278:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204304.617284:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204304.617290:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205849600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204304.617304:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204304.617309:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204304.617318:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205849600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204304.617342:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204304.617349:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5769) 00000800:00000200:2.1:1788204304.617496:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204304.617498:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204304.617718:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204304.617720:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204304.617933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204304.617936:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00010000:00000001:1.0:1788204304.808472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204304.808477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204304.808481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204304.808482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204304.808484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204304.808485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204304.808487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204304.808488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204304.808540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204304.808543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204304.808546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204305.576482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204305.576488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204305.576490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204305.576495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204305.576499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204305.576502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204305.576505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204305.576507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204305.576510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204305.640463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204305.640470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204305.640474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204305.640480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204305.640485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204305.640490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204305.640497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204305.640501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204305.640504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204305.832520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204305.832527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204305.832533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204305.832533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204305.832535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204305.832536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204305.832538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204305.832539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204305.832566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204305.832568:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204305.832570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204306.600502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204306.600510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204306.600512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204306.600517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204306.600521:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204306.600524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204306.600527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204306.600529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204306.600531:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204306.664475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204306.664484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204306.664487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204306.664493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204306.664501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204306.664506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204306.664516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204306.664522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204306.664524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204306.856493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204306.856500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204306.856505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204306.856507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204306.856509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204306.856510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204306.856513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204306.856514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204306.856592:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204306.856595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204306.856598:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204307.624465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204307.624474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204307.624476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204307.624481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204307.624484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204307.624488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204307.624498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204307.624500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204307.624502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204307.688505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204307.688526:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204307.688529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204307.688539:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204307.688550:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204307.688560:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204307.688571:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204307.688576:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204307.688581:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204307.880581:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880589:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204307.880592:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204307.880596:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204307.880600:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880603:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204307.880606:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880608:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880611:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880614:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204307.880615:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204307.880619:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204307.880620:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204307.880655:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204307.880659:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204307.880662:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204308.648531:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204308.648540:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204308.648542:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204308.648547:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204308.648550:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204308.648554:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204308.648556:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204308.648558:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204308.648561:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204308.712471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204308.712487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204308.712490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204308.712495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204308.712512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204308.712517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204308.712529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204308.712533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204308.712537:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204308.904452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204308.904461:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204308.904469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204308.904470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204308.904474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204308.904476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204308.904479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204308.904481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204308.904560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204308.904565:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204308.904569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204309.224474:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300435969, at 4300436480 00000100:00000040:1.0:1788204309.224474:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300435969, at 4300436480 00000100:00000010:1.0:1788204309.224502:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000010:0.0:1788204309.224504:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000001:1.0:1788204309.672444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204309.672450:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204309.672452:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204309.672458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204309.672462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204309.672466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204309.672468:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204309.672472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204309.672475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204309.736471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204309.736491:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204309.736495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204309.736506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204309.736511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204309.736516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204309.736527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204309.736530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204309.736534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.736794:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204309.736799:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204309.736803:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204309.736811:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205849728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.736822:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204309.736825:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.736833:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204309.736836:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204309.736839:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204309.736843:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205849728:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204309.736849:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204309.736854:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204309.736857:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.736860:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204309.736866:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788204309.736869:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204309.736876:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204309.736882:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da058. 00000400:00000010:0.0:1788204309.736891:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192172a8. 00000100:00000200:0.0:1788204309.736900:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205849728, portal 25 00000100:00000001:0.0:1788204309.736903:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204309.736905:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788204309.736912:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205849728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.736919:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204309.736922:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204309.736926:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788204309.736929:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205849728, offset 0 00000400:00000200:0.0:1788204309.736937:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204309.736947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.736954:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204309.736962:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204309.736966:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204309.736971:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204309.736975:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204309.736980:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58494:58494:256:4294967295] 192.168.201.151@tcp LPNI seq info [58474:58474:8:4294967295] 00000400:00000200:0.0:1788204309.736988:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737005:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204309.737019:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204309.737025:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788204309.737035:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204309.737043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204309.737070:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204309.737078:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204309.737083:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300441089 00000100:00000010:0.0:1788204309.737092:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204309.737111:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204309.737125:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300441089 00000100:00000010:0.0:1788204309.737132:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788204309.737139:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204309.737146:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204309.737152:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737157:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737173:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204309.737187:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204309.737190:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788204309.737195:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204309.737198:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204309.737201:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204309.737209:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205849856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737226:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204309.737230:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737235:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204309.737240:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204309.737246:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204309.737252:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205849856:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204309.737262:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204309.737267:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204309.737272:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737275:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204309.737279:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788204309.737283:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204309.737289:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204309.737295:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da0b0. 00000400:00000010:0.0:1788204309.737303:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192173b8. 00000100:00000200:0.0:1788204309.737315:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205849856, portal 10 00000100:00000001:0.0:1788204309.737319:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204309.737321:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788204309.737334:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205849856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737352:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204309.737357:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204309.737361:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217cc0. 00000100:00000200:0.0:1788204309.737394:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205849856, offset 0 00000400:00000200:0.0:1788204309.737400:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737408:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204309.737415:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204309.737422:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204309.737431:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204309.737435:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737441:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58495:58495:255:4294967295] 192.168.201.151@tcp LPNI seq info [58475:58475:7:4294967295] 00000400:00000200:0.0:1788204309.737445:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737452:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204309.737457:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204309.737461:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054afb00. 00000800:00000200:0.0:1788204309.737465:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204309.737469:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204309.737472:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204309.737476:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204309.737493:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204309.737495:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300437147 00000100:00000001:0.0:1788204309.737498:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737500:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300437147 00000100:00000001:0.0:1788204309.737502:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737504:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204309.737506:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737508:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204309.737512:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205849984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737519:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204309.737521:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737523:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204309.737525:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204309.737526:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204309.737529:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205849984:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204309.737533:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204309.737534:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204309.737536:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737537:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204309.737541:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788204309.737543:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204309.737546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204309.737548:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da000. 00000400:00000010:0.0:1788204309.737552:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217c38. 00000100:00000200:0.0:1788204309.737554:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205849984, portal 4 00000100:00000001:0.0:1788204309.737556:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204309.737558:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788204309.737564:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205849984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737571:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204309.737573:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204309.737576:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204309.737579:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205849984, offset 0 00000400:00000200:0.0:1788204309.737582:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737586:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737590:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204309.737592:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204309.737595:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204309.737597:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204309.737599:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737602:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58496:58496:254:4294967295] 192.168.201.151@tcp LPNI seq info [58476:58476:6:4294967295] 00000400:00000200:0.0:1788204309.737606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737612:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204309.737616:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204309.737620:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788204309.737624:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204309.737629:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204309.737632:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204309.737634:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204309.737637:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204309.737639:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300437033 00000100:00000001:0.0:1788204309.737642:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737644:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300437033 00000100:00000001:0.0:1788204309.737646:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737648:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204309.737650:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737652:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737654:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204309.737658:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204309.737659:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788204309.737661:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204309.737662:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204309.737664:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204309.737667:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205850112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737674:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204309.737675:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737678:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204309.737679:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204309.737681:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204309.737684:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205850112:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204309.737687:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204309.737689:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204309.737691:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204309.737692:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204309.737696:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204309.737698:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204309.737701:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204309.737703:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dad68. 00000400:00000010:0.0:1788204309.737707:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217198. 00000100:00000200:0.0:1788204309.737710:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205850112, portal 4 00000100:00000001:0.0:1788204309.737712:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204309.737713:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788204309.737719:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205850112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204309.737726:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204309.737728:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204309.737730:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204309.737733:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205850112, offset 0 00000400:00000200:0.0:1788204309.737737:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737740:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737743:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204309.737747:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204309.737749:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204309.737751:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204309.737754:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204309.737756:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58497:58497:253:4294967295] 192.168.201.151@tcp LPNI seq info [58477:58477:5:4294967295] 00000400:00000200:0.0:1788204309.737760:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204309.737766:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204309.737770:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204309.737774:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204309.737777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204309.737780:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204309.737783:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204309.737786:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204309.737789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204309.737791:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300437039 00000100:00000001:0.0:1788204309.737793:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737795:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300437039 00000100:00000001:0.0:1788204309.737797:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300441089 00000100:00000001:0.0:1788204309.737799:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204309.737801:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737803:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737805:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204309.737807:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204309.737809:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788204309.737811:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204309.737812:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204309.737813:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204309.737815:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204309.928533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204309.928538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204309.928542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204309.928543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204309.928545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204309.928546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204309.928548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204309.928549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204309.928562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204309.928563:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204309.928565:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204310.696491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204310.696498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204310.696500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204310.696505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204310.696508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204310.696511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204310.696513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204310.696515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204310.696517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204310.760530:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204310.760535:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204310.760536:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204310.760540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204310.760542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204310.760544:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204310.760545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204310.760547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204310.760548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204310.952503:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204310.952506:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204310.952510:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952516:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204310.952518:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952521:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952523:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204310.952529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204310.952533:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204310.952534:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204310.952536:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952538:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204310.952539:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952541:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952543:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204310.952547:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204310.952549:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204310.952550:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952552:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204310.952553:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952555:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204310.952557:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204310.952587:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204310.952590:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204310.952593:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204311.720503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204311.721607:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204311.721608:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204311.721612:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204311.721615:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204311.721618:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204311.721619:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204311.721621:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204311.721622:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204311.784562:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204311.784565:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204311.784566:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204311.784568:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204311.784569:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204311.784571:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204311.784572:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204311.784574:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204311.784575:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204311.976553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204311.976558:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204311.976561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204311.976562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204311.976564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204311.976565:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204311.976567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204311.976567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204311.976580:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204311.976582:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204311.976584:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204312.744485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204312.744491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204312.744493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204312.744496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204312.744498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204312.744501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204312.744502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204312.744504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204312.744505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204312.808595:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204312.808599:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204312.808600:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204312.808603:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204312.808605:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204312.808608:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204312.808609:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204312.808611:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204312.808613:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204312.808694:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:0.0:1788204312.808735:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204312.808744:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204312.808748:0:2333:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:0.0:1788204312.808753:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204312.808756:0:2333:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788204312.808761:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204312.808763:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:0.0:1788204312.808769:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204312.808772:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204312.808777:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:0.0:1788204312.808780:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:0.0:1788204312.808784:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204312.808793:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302680 x1875062205849728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808818:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204312.808822:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205849728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808829:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204312.808831:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204312.808840:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204312.808843:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054afb00. 00000400:00000200:0.0:1788204312.808846:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204312.808850:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204312.808853:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217cc0 00000400:00000010:0.0:1788204312.808855:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217cc0. 00000100:00000001:0.0:1788204312.808858:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204312.808875:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310303480 x1875062205849856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808888:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204312.808896:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205849856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808908:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204312.808911:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204312.808919:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204312.808921:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788204312.808924:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204312.808929:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204312.808948:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204312.808952:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204312.808957:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204312.808964:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310301500 x1875062205849984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808977:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204312.808983:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205849984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.808996:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204312.808998:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204312.809007:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204312.809013:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204312.809019:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204312.809025:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204312.809031:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204312.809035:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204312.809039:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204312.809047:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302d80 x1875062205850112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.809066:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204312.809073:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205850112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204312.809085:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204312.809089:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1788204313.000484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204313.000489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204313.000493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204313.000494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204313.000496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204313.000497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204313.000500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204313.000501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204313.000549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204313.000552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204313.000555:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204313.768433:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204313.768440:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204313.768448:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204313.768456:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204313.768459:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204313.768463:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204313.768465:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204313.768468:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204313.768471:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204313.832502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204313.832506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204313.832507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204313.832510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204313.832517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204313.832519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204313.832521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204313.832522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204313.832523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204314.024563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204314.024569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204314.024575:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204314.024576:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204314.024578:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204314.024579:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204314.024581:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204314.024581:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204314.024595:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204314.024597:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204314.024600:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204314.344470:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300441089, at 4300441600 00000100:00000040:0.0:1788204314.344474:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300441089, at 4300441600 00000100:00000010:1.0:1788204314.344486:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000010:0.0:1788204314.344488:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000001:1.0:1788204314.792493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204314.792500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204314.792503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204314.792508:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204314.792511:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204314.792514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204314.792517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204314.792519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204314.792522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204314.856475:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204314.856480:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204314.856482:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204314.856485:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204314.856488:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204314.856494:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5769 00000400:00000200:0.0:1788204314.856502:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204314.856506:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204314.856508:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204314.856511:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204314.856512:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204314.856514:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204314.856515:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204314.856517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.856518:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856521:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:1.0:1788204314.856521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204314.856525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204314.856526:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204314.856528:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:1.0:1788204314.856528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204314.856531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204314.856532:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788204314.856533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788204314.856535:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:1.0:1788204314.856536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204314.856539:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204314.856542:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204314.856544:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856546:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.856547:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856549:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204314.856551:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204314.856553:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856554:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204314.856556:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204314.856558:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204314.856560:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.856561:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204314.856563:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204314.856568:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204314.856570:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204314.856573:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204314.856574:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.856576:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204314.856577:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204314.856580:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204314.856586:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204314.856591:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204314.856606:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205850240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204314.856614:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204314.856617:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.856620:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205850240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204314.856635:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204314.856638:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204314.856641:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204314.856643:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204314.856644:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204314.856647:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204314.856649:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5769 00000400:00000200:0.0:1788204314.856653:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204314.856656:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204314.856659:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204314.856662:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856665:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.856667:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856671:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204314.856676:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204314.856681:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204314.856684:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204314.856686:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204314.856691:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204314.856696:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204314.856705:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856711:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.856716:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856720:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204314.856725:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204314.856732:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856737:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204314.856755:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204314.856762:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204314.856766:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.856769:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204314.856775:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204314.856780:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.856782:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204314.856787:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204314.856792:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204314.856803:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204314.856807:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.856811:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204314.856813:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204314.856817:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204314.856824:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204314.856830:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204314.856839:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205850368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204314.856856:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204314.856861:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.856869:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205850368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204314.856882:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204314.856888:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204314.856895:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204314.856898:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204314.856901:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204314.856907:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204314.856913:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5769 00000400:00000200:0.0:1788204314.856921:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204314.856925:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204314.856927:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204314.856930:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856932:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.856934:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856938:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204314.856942:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204314.856945:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204314.856948:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204314.856950:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204314.856954:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204314.856957:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204314.856963:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856965:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.856969:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856971:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204314.856975:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204314.856982:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204314.856987:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204314.856991:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1788204314.856992:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204314.856995:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1788204314.856997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 02000000:00000001:0.0:1788204314.856999:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857003:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204314.857007:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204314.857011:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857015:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204314.857019:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204314.857024:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204314.857030:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204314.857031:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857034:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204314.857038:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204314.857041:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204314.857047:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204314.857053:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204314.857061:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205850496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204314.857073:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204314.857077:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.857087:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205850496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204314.857098:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204314.857102:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204314.857109:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204314.857112:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204314.857115:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204314.857120:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204314.857125:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5769 00000400:00000200:0.0:1788204314.857130:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204314.857136:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204314.857140:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204314.857146:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.857151:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.857156:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.857161:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204314.857166:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204314.857169:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204314.857172:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204314.857176:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204314.857182:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204314.857186:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204314.857194:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.857205:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204314.857211:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204314.857216:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204314.857220:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204314.857227:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204314.857233:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204314.857236:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204314.857241:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204314.857246:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857251:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204314.857255:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204314.857259:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857263:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204314.857269:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204314.857274:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204314.857279:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204314.857282:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204314.857287:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204314.857289:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204314.857293:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204314.857299:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204314.857304:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204314.857313:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205850624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204314.857327:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204314.857332:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.857340:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205850624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204314.857429:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204314.857437:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5779) 00000800:00000200:2.1:1788204314.857586:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204314.857588:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204314.857859:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204314.857862:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204314.858123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204314.858125:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00010000:00000001:1.0:1788204315.048477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204315.048484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204315.048491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204315.048493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204315.048496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204315.048497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204315.048500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204315.048502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204315.048556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204315.048560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204315.048562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204315.816504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204315.816510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204315.816511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204315.816515:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204315.816517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204315.816520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204315.816522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204315.816524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204315.816526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204315.880534:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204315.880537:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204315.880538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204315.880541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204315.880542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204315.880544:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204315.880545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204315.880546:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204315.880548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204316.072516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204316.072523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204316.072529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204316.072531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204316.072533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204316.072534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204316.072537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204316.072538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204316.072553:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204316.072555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204316.072558:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204316.840467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204316.840473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204316.840474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204316.840479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204316.840482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204316.840485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204316.840486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204316.840488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204316.840490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204316.904434:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204316.904439:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204316.904442:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204316.904445:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204316.904449:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204316.904451:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204316.904452:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204316.904454:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204316.904456:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204317.096487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204317.096496:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204317.096498:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204317.096500:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096503:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204317.096504:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096505:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096507:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204317.096510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204317.096512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204317.096513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204317.096544:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204317.096548:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204317.096551:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204317.864481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204317.864487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204317.864488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204317.864492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204317.864494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204317.864497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204317.864498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204317.864500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204317.864502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204317.928593:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204317.928595:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204317.928596:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204317.928597:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204317.928599:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204317.928600:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204317.928601:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204317.928602:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204317.928603:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204318.120505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204318.120510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204318.120513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204318.120514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204318.120516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204318.120517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204318.120519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204318.120520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204318.120532:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204318.120534:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204318.120536:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204318.888446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204318.888451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204318.888453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204318.888456:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204318.888459:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204318.888461:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204318.888463:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204318.888465:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204318.888466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204318.952458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204318.952465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204318.952468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204318.952475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204318.952480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204318.952485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204318.952492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204318.952496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204318.952499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204319.144462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204319.144470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204319.144475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204319.144476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204319.144478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204319.144479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204319.144481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204319.144482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204319.144525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204319.144529:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204319.144532:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204319.464461:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300446209, at 4300446720 00000100:00000040:1.0:1788204319.464461:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300446209, at 4300446720 00000100:00000010:0.0:1788204319.464477:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000010:1.0:1788204319.464480:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000001:1.0:1788204319.912460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204319.912465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204319.912467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204319.912471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204319.912474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204319.912476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204319.912478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204319.912480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204319.912482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204319.976472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204319.976474:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204319.976480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204319.976481:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204319.976484:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204319.976484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204319.976488:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204319.976490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204319.976494:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204319.976495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204319.976500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204319.976503:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5774 00000100:00000001:1.0:1788204319.976504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204319.976509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204319.976513:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204319.976513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204319.976521:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204319.976524:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204319.976527:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976529:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976531:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976533:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204319.976536:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204319.976537:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204319.976539:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204319.976541:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204319.976543:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204319.976545:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204319.976547:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976548:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976549:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976550:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204319.976552:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204319.976554:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976555:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204319.976557:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204319.976560:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204319.976561:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976563:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204319.976564:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204319.976572:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204319.976574:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204319.976577:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204319.976578:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976581:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204319.976582:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204319.976585:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204319.976593:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204319.976599:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204319.976605:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205850752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204319.976612:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204319.976615:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204319.976618:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205850752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204319.976640:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204319.976642:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204319.976644:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204319.976645:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204319.976646:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204319.976648:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204319.976650:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5774 00000400:00000200:0.0:1788204319.976652:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204319.976653:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204319.976655:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204319.976657:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976658:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976659:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976660:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204319.976661:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204319.976662:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204319.976663:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204319.976664:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204319.976666:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204319.976667:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204319.976669:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976670:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976672:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976673:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204319.976675:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204319.976677:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976678:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204319.976679:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204319.976683:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204319.976684:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976685:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204319.976689:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204319.976692:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976694:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204319.976699:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204319.976703:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204319.976705:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204319.976707:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976708:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204319.976710:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204319.976712:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204319.976716:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204319.976720:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204319.976727:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205850880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204319.976742:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204319.976746:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204319.976750:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205850880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204319.976759:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204319.976763:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204319.976767:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204319.976768:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204319.976770:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204319.976774:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204319.976777:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5774 00000400:00000200:0.0:1788204319.976781:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204319.976784:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204319.976786:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204319.976788:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976793:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976794:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976797:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204319.976798:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204319.976799:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204319.976801:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204319.976803:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204319.976808:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204319.976809:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204319.976811:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976815:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976817:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976818:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204319.976819:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204319.976821:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976825:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204319.976827:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204319.976828:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204319.976830:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976834:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204319.976836:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204319.976840:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976843:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204319.976847:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204319.976849:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204319.976852:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204319.976853:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.976856:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204319.976858:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204319.976860:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204319.976864:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204319.976868:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204319.976873:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205851008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204319.976882:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204319.976884:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204319.976891:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205851008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204319.976899:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204319.976901:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204319.976905:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204319.976908:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204319.976910:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204319.976911:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204319.976913:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5774 00000400:00000200:0.0:1788204319.976919:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204319.976925:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204319.976928:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204319.976934:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976937:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976941:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976945:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204319.976948:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204319.976952:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204319.976954:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204319.976958:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204319.976963:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204319.976968:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204319.976975:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976977:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204319.976980:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976985:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204319.976986:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204319.976988:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204319.976990:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204319.976993:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204319.976996:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204319.977000:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.977002:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204319.977005:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204319.977008:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.977011:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204319.977014:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204319.977017:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204319.977019:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204319.977020:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204319.977021:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204319.977024:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204319.977026:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204319.977030:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204319.977034:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204319.977041:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205851136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1788204319.977047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204319.977052:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1788204319.977053:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204319.977055:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204319.977061:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205851136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204319.977069:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204319.977076:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5784) 00000800:00000200:2.1:1788204319.977332:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204319.977333:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204319.977550:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204319.977551:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204319.977713:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204319.977715:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00010000:00000001:1.0:1788204320.168516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204320.168525:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204320.168527:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204320.168530:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168535:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204320.168537:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168539:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168541:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204320.168547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204320.168551:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204320.168552:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204320.168554:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168555:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204320.168556:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168558:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168559:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168562:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204320.168564:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204320.168565:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204320.168566:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168568:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204320.168569:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168570:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204320.168572:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204320.168589:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204320.168592:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204320.168595:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204320.936454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204320.936461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204320.936463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204320.936468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204320.936471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204320.936475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204320.936477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204320.936480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204320.936483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204321.000455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204321.000461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204321.000464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204321.000471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204321.000477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204321.000480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204321.000483:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204321.000486:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204321.000489:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204321.192466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204321.192473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204321.192480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204321.192481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204321.192485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204321.192486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204321.192490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204321.192491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204321.192541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204321.192545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204321.192547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204321.960470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204321.960475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204321.960477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204321.960481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204321.960483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204321.960486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204321.960488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204321.960489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204321.960491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204322.024439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204322.024446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204322.024449:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204322.024455:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204322.024459:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204322.024463:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204322.024468:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204322.024471:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204322.024473:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204322.216507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204322.216514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204322.216520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204322.216522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204322.216525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204322.216527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204322.216530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204322.216532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204322.216552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204322.216555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204322.216558:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204322.984516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204322.984525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204322.984527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204322.984534:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204322.984539:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204322.984542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204322.984545:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204322.984548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204322.984550:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204323.048498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204323.048509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204323.048512:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204323.048520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204323.048527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204323.048533:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204323.048551:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204323.048554:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204323.048557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204323.240519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204323.240529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204323.240535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204323.240536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204323.240539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204323.240541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204323.240544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204323.240545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204323.240606:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204323.240610:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204323.240614:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204324.008426:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204324.008436:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204324.008438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204324.008444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204324.008447:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204324.008451:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204324.008453:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204324.008456:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204324.008459:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204324.072453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204324.072462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204324.072466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204324.072476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204324.072482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204324.072489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204324.072502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204324.072505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204324.072508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204324.264505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204324.264513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204324.264520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204324.264521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204324.264524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204324.264525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204324.264528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204324.264529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204324.264557:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204324.264560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204324.264562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204324.584505:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300451329, at 4300451840 00000100:00000040:1.0:1788204324.584506:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300451329, at 4300451840 00000100:00000010:0.0:1788204324.584531:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000010:1.0:1788204324.584534:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000001:1.0:1788204325.032490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204325.032499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204325.032501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204325.032507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204325.032510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204325.032515:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204325.032517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204325.032519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204325.032522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204325.096507:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204325.096512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204325.096515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204325.096516:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204325.096523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204325.096524:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204325.096530:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204325.096530:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204325.096537:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204325.096539:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204325.096542:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204325.096549:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204325.096551:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5779 00000100:00000001:1.0:1788204325.096552:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204325.096559:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204325.096567:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204325.096577:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204325.096581:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204325.096584:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.096591:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096594:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204325.096599:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204325.096601:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204325.096604:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204325.096607:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204325.096609:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204325.096611:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204325.096615:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096617:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.096619:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096622:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204325.096626:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204325.096630:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096632:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204325.096635:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204325.096638:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204325.096641:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.096642:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204325.096645:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204325.096662:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204325.096665:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204325.096669:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204325.096671:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.096674:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204325.096677:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204325.096681:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204325.096691:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204325.096698:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204325.096707:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205851264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204325.096716:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204325.096721:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204325.096725:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205851264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204325.096743:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204325.096746:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204325.096749:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204325.096750:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204325.096752:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204325.096754:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204325.096756:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5779 00000400:00000200:0.0:1788204325.096759:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204325.096761:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204325.096763:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204325.096765:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096766:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.096768:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096769:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204325.096770:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204325.096771:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204325.096772:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204325.096774:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204325.096775:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204325.096777:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204325.096781:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096782:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.096786:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096789:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204325.096791:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204325.096795:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204325.096799:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204325.096801:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204325.096808:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204325.096814:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.096820:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204325.096828:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204325.096833:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.096837:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204325.096846:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204325.096850:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204325.096857:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204325.096861:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.096878:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204325.096882:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204325.096887:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204325.096898:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204325.096908:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204325.096921:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205851392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204325.096931:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204325.096936:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204325.096944:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205851392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204325.096973:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204325.096980:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204325.096984:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204325.096996:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204325.097003:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204325.097012:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204325.097024:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5779 00000400:00000200:0.0:1788204325.097040:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204325.097056:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204325.097063:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204325.097076:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097085:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.097092:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097094:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204325.097096:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204325.097101:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204325.097104:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204325.097106:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204325.097108:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204325.097113:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204325.097119:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097125:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.097135:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097146:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204325.097158:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204325.097169:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097180:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204325.097188:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204325.097210:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204325.097240:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204325.097244:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204325.097248:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.1:1788204325.097249:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000008:00100000:0.0:1788204325.097255:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204325.097265:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.097270:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204325.097282:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204325.097291:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204325.097302:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204325.097308:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.097320:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204325.097325:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204325.097330:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204325.097340:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204325.097350:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204325.097391:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205851520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204325.097398:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204325.097400:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204325.097405:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205851520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204325.097423:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204325.097426:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204325.097434:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204325.097438:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204325.097441:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204325.097444:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204325.097449:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5779 00000400:00000200:0.0:1788204325.097464:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204325.097471:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204325.097480:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204325.097493:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097499:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.097509:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097517:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204325.097525:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204325.097532:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204325.097535:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204325.097542:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204325.097553:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204325.097562:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204325.097577:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097581:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204325.097585:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097593:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204325.097600:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204325.097611:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204325.097618:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204325.097630:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204325.097636:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204325.097641:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.097644:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204325.097648:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204325.097653:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.097655:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204325.097660:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204325.097668:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204325.097670:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204325.097673:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204325.097676:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204325.097679:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204325.097682:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204325.097688:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204325.097693:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204325.097704:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205851648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204325.097720:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204325.097725:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204325.097732:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205851648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204325.097783:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204325.097792:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5790) 00000800:00000200:2.1:1788204325.098051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204325.098053:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788204325.098429:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204325.098432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1788204325.098723:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204325.098726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00010000:00000001:1.0:1788204325.288526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204325.288533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204325.288540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204325.288541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204325.288544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204325.288545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204325.288549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204325.288550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204325.288617:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204325.288623:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204325.288628:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204326.056445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204326.056451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204326.056453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204326.056458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204326.056461:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204326.056464:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204326.056466:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204326.056468:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204326.056470:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204326.120523:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204326.120532:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204326.120535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204326.120543:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204326.120550:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204326.120554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204326.120564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204326.120567:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204326.120571:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204326.312532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204326.312540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204326.312547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204326.312548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204326.312550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204326.312551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204326.312554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204326.312554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204326.312570:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204326.312572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204326.312575:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204327.080509:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204327.080517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204327.080519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204327.080525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204327.080529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204327.080533:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204327.080536:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204327.080539:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204327.080541:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204327.144474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204327.144483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204327.144489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204327.144497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204327.144505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204327.144511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204327.144517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204327.144522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204327.144526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204327.336485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204327.336494:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204327.336496:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204327.336499:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336501:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204327.336504:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336505:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336507:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204327.336511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204327.336514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204327.336515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204327.336530:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204327.336536:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204327.336540:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204328.104509:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204328.104516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204328.104517:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204328.104521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204328.104524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204328.104528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204328.104530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204328.104531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204328.104533:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204328.168501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204328.168508:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204328.168512:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204328.168517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204328.168522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204328.168527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204328.168538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204328.168541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204328.168544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204328.360518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204328.360526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204328.360532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204328.360534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204328.360537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204328.360539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204328.360542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204328.360543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204328.360563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204328.360567:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204328.360576:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204329.128465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204329.128473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204329.128475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204329.128482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204329.128486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204329.128490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204329.128493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204329.128496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204329.128499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204329.192493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204329.192501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204329.192505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204329.192512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204329.192518:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204329.192524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204329.192532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204329.192536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204329.192539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204329.384509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204329.384517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204329.384523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204329.384524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204329.384528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204329.384529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204329.384531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204329.384532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204329.384586:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204329.384590:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204329.384593:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204329.704481:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300456450, at 4300456960 00000100:00000040:0.0:1788204329.704491:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300456449, at 4300456960 00000100:00000010:1.0:1788204329.704495:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000010:0.0:1788204329.704506:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000001:1.0:1788204330.152503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204330.152511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204330.152512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204330.152517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204330.152520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204330.152523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204330.152525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204330.152528:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204330.152530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204330.216479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204330.216480:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204330.216490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204330.216491:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204330.216495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216498:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204330.216501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204330.216504:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204330.216508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204330.216511:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204330.216514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204330.216522:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5785 00000100:00000001:1.0:1788204330.216523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204330.216528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204330.216533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204330.216537:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204330.216546:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204330.216549:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204330.216552:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216555:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.216557:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216559:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204330.216563:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204330.216564:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204330.216566:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204330.216568:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204330.216570:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204330.216571:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204330.216574:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216575:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.216576:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216578:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204330.216580:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204330.216582:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216584:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204330.216586:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204330.216589:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204330.216591:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216593:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204330.216595:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204330.216603:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204330.216606:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204330.216609:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204330.216611:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216613:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204330.216616:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204330.216619:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204330.216625:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204330.216631:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204330.216638:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205851776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204330.216646:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204330.216649:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204330.216652:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205851776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204330.216672:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204330.216674:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204330.216677:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204330.216678:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204330.216680:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204330.216689:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204330.216692:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5785 00000400:00000200:0.0:1788204330.216695:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204330.216702:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204330.216705:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204330.216709:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216712:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.216714:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216717:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204330.216721:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204330.216722:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204330.216724:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204330.216727:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204330.216729:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204330.216732:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204330.216738:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216744:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.216751:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216755:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204330.216759:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204330.216768:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204330.216775:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204330.216781:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204330.216788:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204330.216800:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216803:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204330.216809:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204330.216813:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216817:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204330.216822:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204330.216828:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204330.216832:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204330.216835:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.216839:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204330.216843:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204330.216846:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204330.216855:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204330.216862:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204330.216882:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205851904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204330.216903:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204330.216909:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204330.216920:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205851904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204330.216940:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204330.216946:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204330.216954:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204330.216958:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204330.216961:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204330.216966:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204330.216973:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5785 00000400:00000200:0.0:1788204330.216990:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204330.216997:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204330.217000:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204330.217006:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217008:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.217012:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217017:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204330.217020:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204330.217023:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000800:00000200:2.1:1788204330.217023:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:0.0:1788204330.217027:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:2.1:1788204330.217029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000020:00000040:0.0:1788204330.217030:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204330.217035:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204330.217039:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204330.217045:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217049:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.217051:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217055:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204330.217060:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204330.217066:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217070:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204330.217074:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204330.217076:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204330.217079:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217082:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204330.217084:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204330.217086:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217091:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204330.217097:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204330.217101:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204330.217106:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204330.217108:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217112:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204330.217115:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204330.217119:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204330.217156:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204330.217164:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204330.217181:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205852032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204330.217191:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204330.217193:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204330.217197:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205852032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204330.217206:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204330.217208:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204330.217214:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204330.217217:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204330.217220:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204330.217223:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204330.217227:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5785 00000400:00000200:0.0:1788204330.217234:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204330.217239:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204330.217243:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204330.217249:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217251:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.217255:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217259:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204330.217263:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204330.217265:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204330.217267:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204330.217271:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204330.217274:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204330.217278:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204330.217281:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217284:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204330.217289:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217293:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204330.217298:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204330.217305:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204330.217310:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204330.217315:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204330.217320:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204330.217326:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217332:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204330.217336:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204330.217342:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217347:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204330.217356:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204330.217360:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204330.217384:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204330.217385:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204330.217386:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204330.217388:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204330.217390:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204330.217395:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204330.217400:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204330.217410:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205852160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204330.217425:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204330.217428:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204330.217436:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205852160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204330.217452:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204330.217457:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5795) 00000800:00000200:2.1:1788204330.217707:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204330.217710:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204330.217992:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204330.217995:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1788204330.218294:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204330.218297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00010000:00000001:1.0:1788204330.408552:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204330.408560:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204330.408563:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204330.408565:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408570:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204330.408571:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408573:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408575:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408577:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204330.408579:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408581:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204330.408583:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204330.408584:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204330.408585:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408586:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204330.408587:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408588:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408589:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408591:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204330.408592:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204330.408593:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204330.408594:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408595:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204330.408596:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408597:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204330.408598:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204330.408614:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204330.408617:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204330.408619:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204331.176467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204331.176474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204331.176477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204331.176483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204331.176487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204331.176491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204331.176494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204331.176497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204331.176500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204331.240481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204331.240490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204331.240493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204331.240503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204331.240510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204331.240517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204331.240522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204331.240526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204331.240529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204331.432576:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204331.432581:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204331.432585:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204331.432586:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204331.432589:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204331.432590:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204331.432592:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204331.432593:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204331.432654:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204331.432658:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204331.432660:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204332.200488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204332.200496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204332.200499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204332.200505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204332.200507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204332.200511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204332.200513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204332.200514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204332.200517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204332.264459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204332.264469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204332.264473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204332.264479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204332.264486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204332.264492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204332.264500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204332.264504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204332.264507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204332.456541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204332.456548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204332.456554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204332.456555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204332.456558:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204332.456560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204332.456563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204332.456565:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204332.456586:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204332.456589:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204332.456592:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204333.224492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204333.224498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204333.224501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204333.224506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204333.224510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204333.224514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204333.224516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204333.224519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204333.224522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204333.288481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204333.288489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204333.288493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204333.288501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204333.288507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204333.288512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204333.288526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204333.288530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204333.288533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204333.480451:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204333.480457:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204333.480462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204333.480464:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204333.480466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204333.480467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204333.480469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204333.480470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204333.480525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204333.480528:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204333.480531:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204334.248490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204334.248497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204334.248499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204334.248503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204334.248506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204334.248509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204334.248511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204334.248513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204334.248516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204334.312443:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204334.312451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204334.312457:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204334.312464:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204334.312470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204334.312480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204334.312487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204334.312491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204334.312494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204334.504497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204334.504502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204334.504507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204334.504507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204334.504509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204334.504510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204334.504512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204334.504513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204334.504527:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204334.504529:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204334.504531:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204334.824433:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300461569, at 4300462080 00000100:00000040:0.0:1788204334.824439:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300461569, at 4300462080 00000100:00000010:1.0:1788204334.824448:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000010:0.0:1788204334.824451:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:1.0:1788204335.272465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204335.272471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204335.272473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204335.272477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204335.272480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204335.272483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204335.272485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204335.272487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204335.272489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204335.336464:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204335.336468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204335.336474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204335.336476:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204335.336480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336482:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204335.336489:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204335.336489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204335.336497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204335.336498:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204335.336503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204335.336509:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5790 00000100:00000001:1.0:1788204335.336512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204335.336517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204335.336522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204335.336525:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204335.336533:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204335.336537:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204335.336542:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336546:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.336550:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336552:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204335.336556:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204335.336558:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204335.336561:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204335.336565:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204335.336567:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204335.336570:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204335.336573:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336575:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.336577:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336579:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204335.336582:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204335.336586:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336588:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204335.336591:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204335.336595:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204335.336598:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336600:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204335.336611:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204335.336621:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204335.336625:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204335.336629:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204335.336631:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336634:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204335.336637:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204335.336641:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204335.336650:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204335.336657:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204335.336666:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205852288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204335.336677:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204335.336682:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204335.336687:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205852288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204335.336712:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204335.336716:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204335.336720:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204335.336721:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204335.336723:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204335.336726:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204335.336728:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5790 00000400:00000200:0.0:1788204335.336732:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204335.336736:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204335.336738:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204335.336742:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336745:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.336747:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336750:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204335.336753:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204335.336755:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204335.336757:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204335.336759:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204335.336762:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204335.336768:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204335.336775:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336779:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.336784:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336791:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204335.336809:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204335.336819:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204335.336828:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204335.336834:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204335.336841:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204335.336848:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336852:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204335.336859:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204335.336864:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336869:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204335.336878:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204335.336885:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204335.336892:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204335.336896:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.336900:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204335.336904:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204335.336909:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204335.336918:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204335.336926:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204335.336939:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205852416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204335.336967:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204335.336978:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204335.336987:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205852416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1788204335.337004:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204335.337009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1788204335.337014:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204335.337018:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204335.337026:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204335.337030:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204335.337050:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204335.337054:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204335.337058:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5790 00000400:00000200:0.0:1788204335.337069:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204335.337076:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204335.337080:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204335.337089:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337096:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.337103:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337109:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204335.337115:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204335.337119:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204335.337124:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204335.337129:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204335.337134:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204335.337143:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204335.337152:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337157:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.337162:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337167:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204335.337173:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204335.337183:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337191:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204335.337196:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204335.337202:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204335.337210:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337215:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204335.337222:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204335.337228:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337234:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204335.337253:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204335.337260:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204335.337267:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204335.337269:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337272:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204335.337276:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204335.337280:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204335.337289:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204335.337296:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204335.337308:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205852544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204335.337329:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204335.337333:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204335.337341:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205852544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204335.337359:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204335.337397:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204335.337408:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204335.337412:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204335.337416:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204335.337423:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204335.337430:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5790 00000400:00000200:0.0:1788204335.337440:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204335.337448:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204335.337453:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204335.337461:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337466:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.337472:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337477:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204335.337482:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204335.337485:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204335.337490:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204335.337496:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204335.337500:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204335.337506:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204335.337516:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337521:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204335.337529:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337535:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204335.337542:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204335.337552:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204335.337559:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204335.337564:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204335.337571:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204335.337579:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337585:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204335.337590:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204335.337599:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337606:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204335.337613:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204335.337620:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204335.337626:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204335.337630:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204335.337635:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204335.337639:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204335.337643:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204335.337652:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204335.337658:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204335.337664:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205852672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204335.337680:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204335.337686:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204335.337699:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205852672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204335.337728:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204335.337736:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5800) 00000800:00000200:2.1:1788204335.337750:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204335.337753:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204335.337964:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204335.337966:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204335.338213:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204335.338215:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00010000:00000001:1.0:1788204335.528453:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204335.528464:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204335.528472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204335.528473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204335.528477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204335.528478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204335.528482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204335.528483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204335.528541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204335.528546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204335.528550:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204336.296476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204336.296482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204336.296484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204336.296488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204336.296491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204336.296494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204336.296496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204336.296498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204336.296500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204336.360484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204336.360491:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204336.360495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204336.360502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204336.360507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204336.360511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204336.360515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204336.360518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204336.360522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204336.552523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204336.552528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204336.552531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204336.552532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204336.552535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204336.552536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204336.552538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204336.552538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204336.552552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204336.552554:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204336.552556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204337.320493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204337.320501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204337.320503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204337.320509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204337.320512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204337.320517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204337.320520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204337.320522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204337.320525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204337.384516:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204337.384525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204337.384531:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204337.384541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204337.384552:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204337.384558:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204337.384566:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204337.384570:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204337.384573:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204337.576495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204337.576503:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204337.576506:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204337.576508:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576511:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204337.576512:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576514:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576515:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204337.576519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204337.576523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204337.576524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204337.576611:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204337.576617:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204337.576622:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204338.344522:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204338.344528:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204338.344530:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204338.344534:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204338.344537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204338.344540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204338.344542:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204338.344544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204338.344546:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204338.408479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204338.408486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204338.408487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204338.408492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204338.408496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204338.408502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204338.408505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204338.408509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204338.408518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204338.600516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204338.600521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204338.600525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204338.600526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204338.600529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204338.600529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204338.600531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204338.600532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204338.600549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204338.600553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204338.600556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204339.368484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204339.368491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204339.368493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204339.368498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204339.368502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204339.368506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204339.368509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204339.368512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204339.368516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204339.432448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204339.432454:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204339.432458:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204339.432463:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204339.432469:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204339.432473:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204339.432478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204339.432481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204339.432485:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204339.624475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204339.624480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204339.624484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204339.624485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204339.624489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204339.624490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204339.624494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204339.624495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204339.624565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204339.624569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204339.624573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204339.944477:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300466689, at 4300467200 00000100:00000040:1.0:1788204339.944477:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300466689, at 4300467200 00000100:00000010:0.0:1788204339.944495:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000010:1.0:1788204339.944498:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000001:1.0:1788204340.392505:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204340.392511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204340.392512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204340.392520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204340.392523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204340.392527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204340.392529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204340.392531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204340.392533:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204340.456547:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204340.456548:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204340.456559:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204340.456560:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204340.456563:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.456566:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204340.456572:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204340.456573:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204340.456577:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204340.456580:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204340.456582:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204340.456587:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204340.456590:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204340.456594:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5795 00000100:00000001:1.0:1788204340.456595:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204340.456617:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204340.456650:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204340.456657:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204340.456663:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456668:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.456673:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456676:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204340.456684:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204340.456686:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204340.456690:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204340.456694:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204340.456697:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204340.456701:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204340.456705:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456708:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.456712:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456715:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204340.456719:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204340.456723:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456726:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204340.456730:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204340.456736:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204340.456740:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.456744:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204340.456747:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204340.456759:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204340.456765:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204340.456770:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204340.456772:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.456776:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204340.456779:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204340.456785:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204340.456796:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204340.456805:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204340.456818:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205852800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204340.456832:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204340.456838:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204340.456846:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205852800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204340.456880:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204340.456891:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204340.456895:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204340.456897:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204340.456899:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204340.456903:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204340.456909:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5795 00000400:00000200:0.0:1788204340.456916:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204340.456922:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204340.456929:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204340.456940:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456963:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.456979:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.456986:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204340.456991:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204340.456995:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204340.456998:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204340.457002:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204340.457006:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204340.457011:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204340.457018:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457022:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.457025:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457030:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204340.457034:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204340.457041:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457046:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204340.457050:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204340.457056:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204340.457061:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457066:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204340.457073:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204340.457082:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457088:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204340.457095:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204340.457099:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204340.457105:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204340.457109:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457116:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204340.457120:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204340.457124:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204340.457131:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204340.457137:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204340.457148:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205852928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204340.457167:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204340.457174:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204340.457185:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205852928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204340.457212:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204340.457217:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204340.457225:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204340.457231:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204340.457234:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204340.457237:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204340.457242:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5795 00000800:00000200:2.1:1788204340.457248:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000400:00000200:0.0:1788204340.457253:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:2.1:1788204340.457254:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000400:00000200:0.0:1788204340.457264:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204340.457269:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204340.457276:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457282:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.457290:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457295:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204340.457300:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204340.457304:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204340.457307:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204340.457310:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204340.457314:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204340.457319:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204340.457332:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457337:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.457342:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457345:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204340.457350:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204340.457355:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457362:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204340.457405:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204340.457412:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204340.457417:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457422:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204340.457426:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204340.457432:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457436:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204340.457441:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204340.457446:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204340.457450:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204340.457453:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457457:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204340.457459:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204340.457462:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204340.457470:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204340.457478:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204340.457490:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205853056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204340.457513:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204340.457519:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204340.457530:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205853056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204340.457542:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204340.457546:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204340.457551:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204340.457553:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204340.457556:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204340.457564:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204340.457572:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5795 00000400:00000200:0.0:1788204340.457586:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204340.457599:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204340.457608:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204340.457618:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457626:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.457633:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457638:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204340.457643:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204340.457647:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204340.457651:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204340.457656:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204340.457664:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204340.457670:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204340.457682:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457688:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204340.457695:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457703:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204340.457711:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204340.457721:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204340.457731:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204340.457736:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204340.457742:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204340.457750:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457757:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204340.457766:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204340.457773:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457780:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204340.457789:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204340.457796:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204340.457804:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204340.457811:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204340.457819:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204340.457822:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204340.457825:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204340.457835:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204340.457845:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204340.457858:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205853184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204340.457877:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204340.457882:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204340.457893:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205853184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204340.457919:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204340.457928:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5805) 00000800:00000200:2.1:1788204340.458040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204340.458042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204340.458351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204340.458355:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204340.458800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204340.458804:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00010000:00000001:1.0:1788204340.648553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204340.648561:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204340.648564:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204340.648567:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648573:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204340.648575:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648576:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648579:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648582:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204340.648584:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648587:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204340.648588:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204340.648590:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204340.648591:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648593:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204340.648594:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648596:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648598:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648601:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204340.648602:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204340.648604:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204340.648605:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648606:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204340.648608:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648609:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204340.648611:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204340.648629:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204340.648632:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204340.648635:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204341.416451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204341.416459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204341.416461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204341.416466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204341.416469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204341.416473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204341.416476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204341.416478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204341.416482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204341.480467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204341.480477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204341.480481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204341.480504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204341.480510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204341.480514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204341.480519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204341.480521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204341.480524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204341.672512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204341.672520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204341.672527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204341.672529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204341.672533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204341.672535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204341.672538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204341.672540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204341.672603:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204341.672608:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204341.672612:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204341.992533:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204341.992540:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204341.992543:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204341.992548:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204341.992551:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204341.992556:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204341.992559:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204341.992561:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204341.992565:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204342.440497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204342.440504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204342.440507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204342.440514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204342.440518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204342.440522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204342.440525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204342.440527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204342.440530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204342.504472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204342.504479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204342.504483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204342.504487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204342.504492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204342.504504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204342.504513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204342.504518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204342.504524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204342.696531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204342.696539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204342.696545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204342.696547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204342.696551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204342.696552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204342.696556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204342.696557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204342.696574:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204342.696576:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204342.696579:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204343.016482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204343.016489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204343.016492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204343.016497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204343.016501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204343.016505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204343.016508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204343.016511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204343.016514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204343.464516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204343.464524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204343.464527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204343.464533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204343.464535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204343.464538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204343.464540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204343.464542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204343.464544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204343.528482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204343.528490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204343.528492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204343.528498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204343.528504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204343.528510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204343.528516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204343.528521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204343.528526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204343.720445:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204343.720454:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204343.720461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204343.720463:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204343.720467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204343.720469:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204343.720473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204343.720476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204343.720547:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204343.720551:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204343.720555:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204344.040501:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204344.040509:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204344.040512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204344.040517:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204344.040521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204344.040525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204344.040528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204344.040531:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204344.040534:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204344.488476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204344.488484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204344.488486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204344.488491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204344.488496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204344.488500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204344.488503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204344.488505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204344.488508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204344.552465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204344.552473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204344.552476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204344.552481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204344.552489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204344.552499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204344.552508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204344.552512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204344.552515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204344.744540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204344.744548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204344.744554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204344.744555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204344.744559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204344.744560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204344.744564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204344.744565:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204344.744584:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204344.744587:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204344.744590:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204345.064459:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300471810, at 4300472320 00000100:00000040:0.0:1788204345.064474:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300471810, at 4300472320 00000100:00000010:1.0:1788204345.064476:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000010:0.0:1788204345.064488:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000001:1.0:1788204345.512483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204345.512491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204345.512493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204345.512499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204345.512502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204345.512505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.512507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204345.512509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.512511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204345.576473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204345.576477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204345.576481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204345.576485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204345.576488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204345.576493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.576496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204345.576499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.576504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204345.576529:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204345.576534:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204345.576537:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204345.576541:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204345.576546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204345.576549:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.576562:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204345.576565:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204345.576571:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204345.576907:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204345.576910:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204345.577206:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204345.577208:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204345.577380:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204345.577382:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204345.577541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204345.577542:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00010000:00000001:1.0:1788204345.768457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204345.768465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204345.768470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204345.768472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204345.768476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204345.768477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204345.768480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204345.768481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204345.768548:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204345.768552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204345.768555:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204346.536489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204346.536494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204346.536496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204346.536501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204346.536503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204346.536506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204346.536508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204346.536510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204346.536512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204346.600503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204346.600507:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204346.600510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204346.600512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204346.600513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204346.600514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204346.600522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204346.600522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204346.600527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204346.600527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204346.600530:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204346.600531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204346.600542:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204346.600545:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204346.600547:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204346.600547:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204346.600550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204346.600552:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204346.792488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204346.792493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204346.792496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204346.792497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204346.792499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204346.792500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204346.792502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204346.792503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204346.792520:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204346.792521:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204346.792523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204347.560478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204347.560485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204347.560487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204347.560493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204347.560496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204347.560499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204347.560501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204347.560503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204347.560505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204347.624482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204347.624482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204347.624491:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204347.624492:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204347.624494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204347.624495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204347.624503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204347.624507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204347.624508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204347.624513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204347.624517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204347.624517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204347.624523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204347.624527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204347.624531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204347.624531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204347.624534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204347.624537:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204347.816496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204347.816506:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204347.816509:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204347.816513:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816524:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204347.816526:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816529:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816531:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204347.816536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204347.816539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204347.816540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204347.816599:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204347.816603:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204347.816607:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204348.584507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204348.584517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204348.584519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204348.584525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204348.584529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204348.584534:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204348.584537:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204348.584540:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204348.584543:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204348.648509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204348.648509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204348.648520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204348.648520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204348.648524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204348.648524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204348.648531:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204348.648532:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204348.648537:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204348.648538:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204348.648541:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204348.648542:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204348.648545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204348.648548:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204348.648551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204348.648553:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204348.648557:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204348.648560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204348.840568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204348.840579:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204348.840587:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204348.840588:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204348.840592:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204348.840594:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204348.840597:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204348.840599:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204348.840624:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204348.840626:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204348.840629:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204349.608517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204349.608525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204349.608527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204349.608532:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204349.608535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204349.608540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204349.608542:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204349.608544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204349.608547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204349.672537:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204349.672549:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204349.672552:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204349.672559:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204349.672564:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204349.672576:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204349.672580:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204349.672583:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204349.672587:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204349.672701:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204349.672706:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204349.672707:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204349.672710:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204349.672712:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204349.672715:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204349.672717:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204349.672718:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204349.672721:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204349.864508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204349.864517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204349.864526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204349.864527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204349.864531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204349.864532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204349.864536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204349.864537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204349.864592:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204349.864596:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204349.864599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204350.184455:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300476929, at 4300477440 00000100:00000040:1.0:1788204350.184455:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300476929, at 4300477440 00000100:00000010:1.0:1788204350.184471:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000010:0.0:1788204350.184476:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00000001:1.0:1788204350.632510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204350.632521:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204350.632524:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204350.632531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204350.632536:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204350.632541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204350.632544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204350.632547:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204350.632551:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204350.696485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204350.696493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204350.696496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204350.696501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204350.696512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204350.696516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204350.696521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204350.696523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204350.696525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204350.696623:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204350.696628:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204350.696629:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204350.696633:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204350.696638:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204350.696641:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204350.696646:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204350.696650:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204350.696656:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204350.697191:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204350.697198:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204350.697714:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204350.697716:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204350.697905:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204350.697907:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204350.698096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204350.698098:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204350.888556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204350.888569:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204350.888572:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204350.888576:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888582:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204350.888585:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888587:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888590:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888596:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204350.888597:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888601:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204350.888602:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204350.888604:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204350.888605:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888607:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204350.888609:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888610:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888612:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888617:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204350.888619:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204350.888620:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204350.888622:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888624:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204350.888625:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888626:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204350.888628:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204350.888661:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204350.888664:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204350.888667:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204351.656500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204351.656510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204351.656514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204351.656521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204351.656526:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204351.656531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204351.656534:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204351.656537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204351.656540:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204351.720466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204351.720466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204351.720476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204351.720476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204351.720479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204351.720479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204351.720486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204351.720491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204351.720493:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204351.720495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204351.720497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204351.720497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204351.720501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204351.720504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204351.720507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204351.720507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204351.720508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204351.720510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204351.912503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204351.912513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204351.912522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204351.912523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204351.912527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204351.912529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204351.912533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204351.912534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204351.912599:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204351.912604:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204351.912607:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204352.680508:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204352.680516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204352.680518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204352.680524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204352.680527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204352.680530:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204352.680533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204352.680535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204352.680538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204352.744493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204352.744493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204352.744501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204352.744502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204352.744506:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204352.744507:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204352.744515:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204352.744515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204352.744522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204352.744522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204352.744529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204352.744529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204352.744535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204352.744539:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204352.744543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204352.744544:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204352.744547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204352.744550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204352.936533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204352.936540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204352.936546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204352.936548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204352.936551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204352.936552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204352.936555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204352.936556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204352.936577:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204352.936580:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204352.936583:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204353.704481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204353.704487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204353.704490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204353.704495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204353.704499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204353.704502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204353.704505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204353.704507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204353.704511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204353.768460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204353.768460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204353.768467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204353.768470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204353.768470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204353.768473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204353.768478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204353.768479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204353.768486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204353.768486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204353.768492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204353.768493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204353.768497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204353.768501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204353.768505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204353.768505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204353.768509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204353.768514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204353.960519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204353.960527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204353.960536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204353.960538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204353.960541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204353.960542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204353.960546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204353.960547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204353.960609:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204353.960613:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204353.960617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204354.728487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204354.728495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204354.728497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204354.728503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204354.728507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204354.728510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204354.728513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204354.728516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204354.728518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204354.792476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204354.792476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204354.792485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204354.792485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204354.792490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204354.792490:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204354.792497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204354.792497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204354.792502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204354.792506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204354.792509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204354.792511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204354.792512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204354.792514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204354.792517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204354.792519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204354.792522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204354.792524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204354.984527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204354.984534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204354.984539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204354.984540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204354.984544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204354.984545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204354.984548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204354.984549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204354.984569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204354.984572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204354.984574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204355.304511:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300482049, at 4300482560 00000100:00000040:1.0:1788204355.304511:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300482050, at 4300482560 00000100:00000010:0.0:1788204355.304535:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00000010:1.0:1788204355.304536:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000001:1.0:1788204355.752471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204355.752478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204355.752480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204355.752485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204355.752489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204355.752492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204355.752495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204355.752497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204355.752500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204355.816447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204355.816455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204355.816458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204355.816464:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204355.816468:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204355.816472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204355.816476:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204355.816480:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204355.816485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204355.817352:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204355.817356:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204355.817863:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204355.817866:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204355.818174:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204355.818177:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204355.818473:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204355.818475:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204356.008493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204356.008499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204356.008504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204356.008506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204356.008509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204356.008510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204356.008512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204356.008513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204356.008578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204356.008581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204356.008584:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204356.776460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204356.776466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204356.776468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204356.776472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204356.776474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204356.776477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204356.776479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204356.776481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204356.776482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204356.840469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204356.840479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204356.840484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204356.840492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204356.840497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204356.840507:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204356.840515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204356.840518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204356.840522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204357.032486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204357.032493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204357.032499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204357.032500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204357.032503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204357.032504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204357.032507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204357.032508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204357.032526:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204357.032529:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204357.032532:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204357.800479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204357.800485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204357.800487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204357.800491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204357.800493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204357.800497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204357.800498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204357.800500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204357.800502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204357.864511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204357.864517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204357.864520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204357.864525:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204357.864529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204357.864532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204357.864536:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204357.864542:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204357.864547:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204358.056467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204358.056479:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204358.056481:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204358.056484:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056486:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204358.056489:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056490:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056492:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204358.056496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204358.056498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204358.056499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204358.056531:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204358.056535:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204358.056538:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204358.824469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204358.824475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204358.824478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204358.824483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204358.824486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204358.824489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204358.824492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204358.824494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204358.824497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204358.888463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204358.888471:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204358.888474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204358.888478:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204358.888483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204358.888488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204358.888497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204358.888500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204358.888502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204359.080504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204359.080509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204359.080515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204359.080517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204359.080520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204359.080521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204359.080524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204359.080525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204359.080542:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204359.080545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204359.080548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204359.848493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204359.848501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204359.848503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204359.848508:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204359.848512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204359.848516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204359.848518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204359.848520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204359.848523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204359.912473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204359.912479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204359.912484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204359.912492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204359.912498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204359.912504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204359.912519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204359.912522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204359.912525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204360.104486:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204360.104489:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204360.104492:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104497:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204360.104499:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104501:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104503:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204360.104514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204360.104518:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204360.104519:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204360.104521:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104523:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204360.104524:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104526:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104527:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204360.104532:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204360.104533:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204360.104534:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104536:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204360.104538:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104539:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204360.104541:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204360.104591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204360.104593:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204360.104596:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204360.424457:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300487170, at 4300487680 00000100:00000040:1.0:1788204360.424457:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300487170, at 4300487680 00000100:00000010:0.0:1788204360.424472:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000010:1.0:1788204360.424474:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000001:1.0:1788204360.872482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204360.872489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204360.872491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204360.872496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204360.872500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204360.872504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204360.872506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204360.872509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204360.872512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204360.936450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204360.936458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204360.936461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204360.936475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204360.936481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204360.936486:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204360.936493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204360.936496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204360.936501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204360.937057:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204360.937064:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204360.937504:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204360.937507:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204360.937793:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204360.937795:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204360.938072:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204360.938075:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00010000:00000001:1.0:1788204361.128525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204361.128529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204361.128533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204361.128534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204361.128536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204361.128537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204361.128539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204361.128539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204361.128551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204361.128553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204361.128555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204361.896507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204361.896515:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204361.896517:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204361.896523:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204361.896527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204361.896531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204361.896533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204361.896536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204361.896539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204361.960483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204361.960491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204361.960500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204361.960506:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204361.960512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204361.960516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204361.960526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204361.960529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204361.960532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204362.152488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204362.152495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204362.152501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204362.152503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204362.152506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204362.152507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204362.152512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204362.152513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204362.152586:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204362.152589:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204362.152593:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204362.920474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204362.920481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204362.920483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204362.920489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204362.920492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204362.920496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204362.920499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204362.920501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204362.920504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204362.984444:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204362.984452:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204362.984455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204362.984455:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204362.984459:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204362.984462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204362.984462:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204362.984466:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204362.984467:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204362.984470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204362.984472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204362.984473:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204362.984477:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204362.984481:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204362.984485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204362.984485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204362.984489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204362.984492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204363.176504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204363.176515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204363.176520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204363.176521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204363.176523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204363.176524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204363.176527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204363.176528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204363.176545:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204363.176548:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204363.176560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204363.496496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204363.496505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204363.496508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204363.496513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204363.496516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204363.496520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204363.496523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204363.496525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204363.496527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204363.944501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204363.944508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204363.944510:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204363.944516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204363.944519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204363.944523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204363.944525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204363.944528:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204363.944530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204364.008480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204364.008483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204364.008487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204364.008489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204364.008491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204364.008492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204364.008500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204364.008500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204364.008506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204364.008506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204364.008512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204364.008512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204364.008516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204364.008518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204364.008521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204364.008526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204364.008528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204364.008531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204364.200475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204364.200482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204364.200488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204364.200489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204364.200492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204364.200493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204364.200497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204364.200498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204364.200564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204364.200567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204364.200569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204364.520412:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204364.520418:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204364.520419:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204364.520423:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204364.520425:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204364.520428:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204364.520430:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204364.520432:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204364.520435:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204364.968511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204364.968519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204364.968547:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204364.968552:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204364.968555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204364.968559:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204364.968561:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204364.968564:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204364.968566:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204365.032468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204365.032469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204365.032477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204365.032477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204365.032481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204365.032482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204365.032488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204365.032489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204365.032493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204365.032497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204365.032502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204365.032502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204365.032504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204365.032507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204365.032508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204365.032514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204365.032516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204365.032530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204365.224500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204365.224505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204365.224508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204365.224510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204365.224512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204365.224513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204365.224524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204365.224526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204365.224539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204365.224541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204365.224542:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204365.544479:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300492289, at 4300492800 00000100:00000040:0.0:1788204365.544493:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300492289, at 4300492800 00000100:00000010:1.0:1788204365.544495:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000010:0.0:1788204365.544504:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000001:1.0:1788204365.992494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204365.992521:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204365.992523:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204365.992527:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204365.992530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204365.992533:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204365.992535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204365.992537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204365.992538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204366.056492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204366.056493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204366.056500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204366.056510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204366.056513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204366.056516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204366.056520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204366.056521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204366.056526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204366.056529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204366.056531:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204366.056534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204366.056536:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204366.056539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204366.056544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204366.056545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204366.056548:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204366.056551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204366.057140:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204366.057146:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204366.057771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204366.057774:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204366.057979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204366.057980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204366.058159:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204366.058161:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00010000:00000001:1.0:1788204366.248468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204366.248475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204366.248481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204366.248483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204366.248486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204366.248488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204366.248492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204366.248493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204366.248593:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204366.248597:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204366.248600:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204366.568643:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204366.568649:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204366.568651:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204366.568656:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204366.568659:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204366.568663:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204366.568666:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204366.568668:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204366.568670:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204367.016528:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204367.016535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204367.016537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204367.016542:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204367.016545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204367.016549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204367.016551:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204367.016554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204367.016556:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204367.080464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204367.080464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204367.080470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204367.080471:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204367.080474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204367.080475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204367.080481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204367.080481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204367.080495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204367.080497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204367.080500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204367.080500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204367.080504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204367.080506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204367.080509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204367.080509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204367.080512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204367.080514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204367.272464:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204367.272469:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204367.272472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204367.272473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204367.272475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204367.272476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204367.272478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204367.272479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204367.272516:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204367.272518:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204367.272520:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204367.592496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204367.592506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204367.592508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204367.592514:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204367.592518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204367.592522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204367.592525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204367.592527:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204367.592530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204368.040509:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204368.040517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204368.040519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204368.040524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204368.040527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204368.040531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204368.040534:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204368.040535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204368.040538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204368.104510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204368.104510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204368.104520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204368.104520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204368.104524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204368.104525:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204368.104531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204368.104532:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204368.104537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204368.104538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204368.104541:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204368.104542:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204368.104545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204368.104549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204368.104551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204368.104555:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204368.104559:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204368.104563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204368.296534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204368.296545:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204368.296548:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204368.296553:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296556:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204368.296558:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296561:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296563:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204368.296569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204368.296572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204368.296573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204368.296642:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204368.296648:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204368.296651:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204368.616455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204368.616474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204368.616476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204368.616481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204368.616485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204368.616489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204368.616491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204368.616494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204368.616496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204369.064482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204369.064490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204369.064492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204369.064497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204369.064500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204369.064504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204369.064505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204369.064507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204369.064510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204369.128531:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204369.128541:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204369.128544:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204369.128550:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204369.128555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204369.128560:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204369.128563:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204369.128567:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204369.128571:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204369.128657:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204369.128659:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204369.128660:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204369.128663:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204369.128664:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204369.128666:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204369.128667:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204369.128668:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204369.128670:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204369.320562:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204369.320572:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204369.320580:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204369.320582:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204369.320585:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204369.320586:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204369.320589:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204369.320590:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204369.320610:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204369.320613:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204369.320615:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204369.640435:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204369.640453:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204369.640455:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204369.640460:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204369.640463:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204369.640467:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204369.640469:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204369.640472:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204369.640475:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204370.088478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204370.088487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204370.088489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204370.088494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204370.088497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204370.088502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204370.088504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204370.088506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204370.088509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204370.152512:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204370.152512:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204370.152523:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204370.152523:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204370.152526:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204370.152534:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204370.152537:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204370.152538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204370.152542:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204370.152545:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204370.152547:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204370.152550:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204370.152552:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204370.152555:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204370.152560:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204370.152561:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204370.152565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204370.152568:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204370.344507:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204370.344510:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204370.344519:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344524:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204370.344526:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344528:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344530:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204370.344534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204370.344538:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204370.344538:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204370.344540:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344541:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204370.344541:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344542:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344544:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204370.344547:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204370.344548:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204370.344549:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344549:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204370.344550:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344551:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204370.344552:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204370.344613:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204370.344618:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204370.344621:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204370.664475:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300497409, at 4300497920 00000100:00000040:1.0:1788204370.664475:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300497409, at 4300497920 00000100:00000010:1.0:1788204370.664495:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00000010:0.0:1788204370.664502:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00000001:1.0:1788204371.112524:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204371.112534:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204371.112537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204371.112544:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204371.112548:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204371.112553:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204371.112556:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204371.112559:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204371.112563:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204371.176502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204371.176502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204371.176512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204371.176512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204371.176516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204371.176517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204371.176524:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204371.176524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204371.176531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204371.176531:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204371.176537:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204371.176537:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204371.176543:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204371.176547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204371.176551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204371.176552:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204371.176556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204371.176560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204371.177059:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204371.177062:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204371.177687:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204371.177689:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204371.177938:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204371.177940:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204371.178310:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204371.178313:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00010000:00000001:1.0:1788204371.368540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204371.368547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204371.368552:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204371.368553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204371.368555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204371.368556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204371.368559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204371.368560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204371.368576:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204371.368578:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204371.368581:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204372.136507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204372.136514:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204372.136516:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204372.136521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204372.136524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204372.136528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204372.136530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204372.136532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204372.136534:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204372.200484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204372.200491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204372.200493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204372.200497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204372.200501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204372.200501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204372.200505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204372.200510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204372.200511:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204372.200512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204372.200517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204372.200517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204372.200527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204372.200533:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204372.200536:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204372.200539:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204372.200542:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204372.200546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204372.392505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204372.392511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204372.392515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204372.392516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204372.392521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204372.392523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204372.392525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204372.392527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204372.392597:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204372.392601:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204372.392604:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204373.160505:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204373.160512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204373.160514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204373.160520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204373.160523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204373.160526:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204373.160529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204373.160532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204373.160535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204373.224445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204373.224446:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204373.224452:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204373.224457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204373.224461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204373.224464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204373.224469:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204373.224470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204373.224474:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204373.224477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204373.224480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204373.224481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204373.224487:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204373.224491:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204373.224494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204373.224495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204373.224498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204373.224500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204373.416571:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204373.416577:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204373.416583:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204373.416584:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204373.416587:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204373.416589:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204373.416592:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204373.416593:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204373.416617:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204373.416620:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204373.416623:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204374.184497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204374.184505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204374.184507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204374.184513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204374.184516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204374.184521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204374.184523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204374.184526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204374.184529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204374.248517:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204374.248518:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204374.248529:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204374.248530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204374.248537:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204374.248538:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204374.248550:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204374.248551:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204374.248563:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204374.248564:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204374.248571:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204374.248571:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204374.248582:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204374.248587:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204374.248592:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204374.248592:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204374.248598:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204374.248601:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204374.440482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204374.440489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204374.440495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204374.440496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204374.440499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204374.440500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204374.440503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204374.440505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204374.440568:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204374.440572:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204374.440575:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204375.208485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204375.208492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204375.208494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204375.208499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204375.208503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204375.208506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204375.208509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204375.208511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204375.208514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.272466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204375.272478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204375.272480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204375.272485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204375.272488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204375.272492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204375.272494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204375.272497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204375.272499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204375.272581:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204375.272583:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204375.272584:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204375.272586:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204375.272587:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204375.272588:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204375.272590:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204375.272591:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204375.272592:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204375.464573:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204375.464580:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204375.464586:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204375.464588:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204375.464591:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204375.464592:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204375.464595:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204375.464596:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204375.464617:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204375.464619:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204375.464621:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204375.784505:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300502529, at 4300503040 00000100:00000040:1.0:1788204375.784505:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300502529, at 4300503040 00000100:00000010:1.0:1788204375.784531:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00000010:0.0:1788204375.784535:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00000001:1.0:1788204376.232527:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204376.232533:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204376.232535:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204376.232539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204376.232542:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204376.232545:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204376.232547:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204376.232548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204376.232550:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204376.296484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204376.296494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204376.296495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204376.296499:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204376.296506:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204376.296509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204376.296513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204376.296515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204376.296518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204376.296524:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204376.296525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204376.296535:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204376.296536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204376.296541:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204376.296542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204376.296545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204376.296549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204376.296554:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204376.297197:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204376.297724:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204376.297875:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204376.298043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00010000:00000001:1.0:1788204376.488491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204376.488499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204376.488505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204376.488506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204376.488511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204376.488512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204376.488515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204376.488517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204376.488570:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204376.488573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204376.488576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204377.256468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204377.256475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204377.256477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204377.256483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204377.256486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204377.256490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204377.256492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204377.256495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204377.256498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204377.320509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204377.320516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204377.320519:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204377.320524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204377.320527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204377.320531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204377.320534:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204377.320536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204377.320539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204377.320617:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204377.320628:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204377.320630:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204377.320635:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204377.320639:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204377.320642:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204377.320645:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204377.320648:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204377.320651:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204377.512493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204377.512500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204377.512505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204377.512506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204377.512509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204377.512510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204377.512512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204377.512513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204377.512530:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204377.512532:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204377.512534:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204378.280546:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204378.280558:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204378.280560:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204378.280568:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204378.280572:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204378.280577:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204378.280580:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204378.280583:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204378.280586:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204378.344471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204378.344485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204378.344488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204378.344496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204378.344501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204378.344506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204378.344509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204378.344513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204378.344517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204378.344543:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204378.344549:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204378.344552:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204378.344557:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204378.344559:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204378.344562:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204378.344564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204378.344565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204378.344567:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204378.536510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204378.536520:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204378.536523:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204378.536527:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536531:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204378.536533:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536535:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536537:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204378.536540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204378.536543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204378.536544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204378.536597:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204378.536601:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204378.536604:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204379.304485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204379.304491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204379.304492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204379.304496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204379.304498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204379.304501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204379.304503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204379.304505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204379.304506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204379.368461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204379.368469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204379.368471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204379.368477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204379.368480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204379.368484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204379.368487:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204379.368489:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204379.368491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204379.368575:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204379.368578:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204379.368579:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204379.368582:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204379.368584:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204379.368586:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204379.368588:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204379.368589:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204379.368591:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204379.560556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204379.560561:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204379.560565:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204379.560566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204379.560567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204379.560568:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204379.560570:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204379.560571:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204379.560585:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204379.560587:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204379.560589:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204380.328453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204380.328461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204380.328463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204380.328467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204380.328469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204380.328472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204380.328474:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204380.328475:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204380.328477:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204380.392468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204380.392474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204380.392475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204380.392479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204380.392482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204380.392485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204380.392487:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204380.392489:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204380.392491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204380.392519:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204380.392524:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204380.392525:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204380.392528:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204380.392530:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204380.392532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204380.392534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204380.392536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204380.392538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204380.584505:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204380.584508:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204380.584513:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584518:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204380.584520:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584523:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584525:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204380.584529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204380.584534:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204380.584536:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204380.584538:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584539:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204380.584540:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584542:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584543:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204380.584546:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204380.584546:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204380.584547:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584548:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204380.584549:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584549:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204380.584551:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204380.584617:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204380.584621:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204380.584624:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204380.904502:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300507649, at 4300508160 00000100:00000040:1.0:1788204380.904502:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300507649, at 4300508160 00000100:00000010:0.0:1788204380.904520:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000010:1.0:1788204380.904525:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000001:1.0:1788204381.352490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204381.352502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204381.352504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204381.352509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204381.352513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204381.352516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204381.352519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204381.352521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204381.352524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204381.416506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204381.416512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204381.416515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204381.416522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204381.416528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204381.416533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204381.416539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204381.416543:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204381.416557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204381.416583:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204381.416592:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204381.416597:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204381.416603:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204381.416609:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204381.416613:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204381.416619:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204381.416624:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204381.416638:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204381.417169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204381.417176:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204381.419901:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204381.419903:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204381.420139:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204381.420141:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204381.420349:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204381.420350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00010000:00000001:1.0:1788204381.608517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204381.608524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204381.608530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204381.608531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204381.608534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204381.608535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204381.608538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204381.608540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204381.608557:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204381.608559:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204381.608562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204382.376491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204382.376499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204382.376501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204382.376507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204382.376510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204382.376514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204382.376517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204382.376519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204382.376522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204382.440478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204382.440486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204382.440489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204382.440494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204382.440498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204382.440502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204382.440504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204382.440507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204382.440510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204382.440563:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204382.440567:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204382.440569:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204382.440571:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204382.440573:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204382.440576:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204382.440578:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204382.440579:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204382.440582:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204382.632465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204382.632471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204382.632477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204382.632479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204382.632482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204382.632483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204382.632487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204382.632488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204382.632548:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204382.632553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204382.632556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204383.400489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204383.400495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204383.400497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204383.400502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204383.400505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204383.400509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204383.400511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204383.400513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204383.400516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204383.464464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204383.464466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204383.464472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204383.464474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204383.464476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204383.464478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204383.464484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204383.464485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204383.464491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204383.464495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204383.464496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204383.464499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204383.464501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204383.464505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204383.464508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204383.464509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204383.464513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204383.464516:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204383.656511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204383.656516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204383.656520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204383.656521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204383.656523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204383.656524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204383.656525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204383.656526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204383.656539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204383.656540:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204383.656542:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204384.424466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204384.424471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204384.424473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204384.424476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204384.424478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204384.424482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204384.424483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204384.424486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204384.424488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204384.488481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204384.488481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204384.488490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204384.488491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204384.488494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204384.488495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204384.488502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204384.488503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204384.488509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204384.488510:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204384.488513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204384.488514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204384.488517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204384.488520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204384.488522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204384.488527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204384.488529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204384.488532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204384.680487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204384.680494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204384.680499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204384.680501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204384.680504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204384.680506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204384.680508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204384.680509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204384.680576:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204384.680581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204384.680585:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204385.448476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204385.448483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204385.448484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204385.448488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204385.448491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204385.448494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204385.448496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204385.448497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204385.448499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204385.512415:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204385.512415:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204385.512421:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204385.512423:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204385.512424:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204385.512427:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204385.512428:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204385.512431:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204385.512432:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204385.512435:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204385.512437:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204385.512438:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204385.512441:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204385.512442:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204385.512443:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204385.512448:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204385.512459:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204385.512463:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204385.704506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204385.704514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204385.704520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204385.704522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204385.704526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204385.704528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204385.704531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204385.704532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204385.704552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204385.704554:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204385.704557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204386.024462:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300512769, at 4300513280 00000100:00000040:0.0:1788204386.024463:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300512769, at 4300513280 00000100:00000010:1.0:1788204386.024478:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000010:0.0:1788204386.024480:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000001:1.0:1788204386.472475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204386.472481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204386.472483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204386.472488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204386.472498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204386.472502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204386.472505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204386.472507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204386.472510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204386.536476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204386.536481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204386.536485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204386.536485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204386.536490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204386.536490:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204386.536497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204386.536503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204386.536505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204386.536508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204386.536510:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204386.536512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204386.536514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204386.536517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204386.536523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204386.536524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204386.536529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204386.536532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204386.537068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204386.537073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204386.537619:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204386.537622:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204386.537861:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204386.537862:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204386.538045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204386.538047:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00010000:00000001:1.0:1788204386.728453:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204386.728459:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204386.728465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204386.728466:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204386.728477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204386.728478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204386.728481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204386.728483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204386.728539:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204386.728543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204386.728546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204387.496453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204387.496459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204387.496461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204387.496464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204387.496466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204387.496470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204387.496472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204387.496474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204387.496477:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204387.560468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204387.560468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204387.560475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204387.560475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204387.560478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204387.560479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204387.560484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204387.560489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204387.560489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204387.560494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204387.560495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204387.560497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204387.560499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204387.560504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204387.560508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204387.560508:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204387.560512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204387.560516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204387.752508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204387.752513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204387.752516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204387.752517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204387.752519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204387.752520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204387.752522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204387.752523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204387.752540:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204387.752541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204387.752543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204388.520429:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204388.520435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204388.520436:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204388.520439:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204388.520451:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204388.520453:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204388.520455:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204388.520457:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204388.520459:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204388.584469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204388.584469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204388.584475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204388.584476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204388.584477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204388.584479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204388.584482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204388.584485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204388.584486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204388.584489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204388.584491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204388.584492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204388.584496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204388.584496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204388.584502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204388.584503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204388.584506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204388.584509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204388.776481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204388.776490:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204388.776493:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204388.776496:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776499:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204388.776501:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776505:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204388.776510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204388.776513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204388.776514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204388.776575:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204388.776579:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204388.776582:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204389.544449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204389.544456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204389.544458:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204389.544463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204389.544466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204389.544470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204389.544472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204389.544474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204389.544477:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204389.608484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204389.608484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204389.608493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204389.608493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204389.608496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204389.608497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204389.608505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204389.608506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204389.608512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204389.608512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204389.608518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204389.608518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204389.608523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204389.608527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204389.608531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204389.608532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204389.608535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204389.608538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204389.800527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204389.800534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204389.800539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204389.800541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204389.800544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204389.800545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204389.800548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204389.800549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204389.800565:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204389.800568:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204389.800570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204390.568470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204390.568478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204390.568480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204390.568485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204390.568488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204390.568490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204390.568492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204390.568493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204390.568496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204390.632485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204390.632485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204390.632494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204390.632494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204390.632496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204390.632497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204390.632504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204390.632504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204390.632510:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204390.632511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204390.632516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204390.632517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204390.632522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204390.632526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204390.632529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204390.632530:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204390.632533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204390.632536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204390.824492:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204390.824494:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204390.824498:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824502:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204390.824505:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824507:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824509:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204390.824514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204390.824519:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204390.824520:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204390.824522:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824523:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204390.824525:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824526:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824528:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204390.824533:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204390.824535:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204390.824536:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824538:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204390.824539:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824540:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204390.824542:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204390.824596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204390.824599:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204390.824601:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204391.144442:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300517889, at 4300518400 00000100:00000040:0.0:1788204391.144447:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300517889, at 4300518400 00000100:00000010:1.0:1788204391.144456:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00000010:0.0:1788204391.144458:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000001:1.0:1788204391.592480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204391.592487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204391.592490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204391.592495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204391.592498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204391.592502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204391.592504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204391.592507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204391.592509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204391.656474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204391.656474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204391.656479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204391.656480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204391.656482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204391.656484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204391.656489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204391.656490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204391.656493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204391.656495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204391.656497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204391.656499:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204391.656500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204391.656504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204391.656507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204391.656509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204391.656513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204391.656516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204391.657020:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204391.657038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204391.657495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204391.657497:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204391.657778:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204391.657781:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204391.658040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204391.658042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00010000:00000001:1.0:1788204391.848641:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204391.848647:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204391.848653:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204391.848654:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204391.848657:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204391.848659:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204391.848661:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204391.848662:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204391.848675:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204391.848677:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204391.848679:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204392.616476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204392.616482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204392.616483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204392.616486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204392.616489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204392.616492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204392.616494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204392.616496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204392.616497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204392.680462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204392.680462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204392.680469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204392.680470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204392.680473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204392.680480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204392.680481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204392.680483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204392.680486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204392.680489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204392.680490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204392.680493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204392.680495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204392.680497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204392.680501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204392.680505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204392.680509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204392.680511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204392.872481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204392.872488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204392.872494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204392.872496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204392.872499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204392.872502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204392.872505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204392.872506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204392.872561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204392.872564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204392.872567:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204393.192453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204393.192461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204393.192462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204393.192467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204393.192471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204393.192475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204393.192478:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204393.192481:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204393.192483:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204393.640467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204393.640474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204393.640475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204393.640479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204393.640481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204393.640484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204393.640486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204393.640488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204393.640489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204393.704473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204393.704481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204393.704483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204393.704490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204393.704494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204393.704498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204393.704501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204393.704504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204393.704507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204393.704509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204393.704514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204393.704516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204393.704519:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204393.704522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204393.704525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204393.704527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204393.704529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204393.704532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204393.896491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204393.896496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204393.896499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204393.896500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204393.896502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204393.896503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204393.896504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204393.896505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204393.896522:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204393.896524:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204393.896526:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204394.216485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204394.216492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204394.216494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204394.216497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204394.216500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204394.216502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204394.216505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204394.216507:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204394.216510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204394.664485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204394.664493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204394.664495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204394.664500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204394.664504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204394.664508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204394.664511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204394.664514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204394.664517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204394.728438:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204394.728439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204394.728444:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204394.728446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204394.728447:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204394.728451:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204394.728452:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204394.728457:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204394.728458:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204394.728460:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204394.728464:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204394.728465:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204394.728466:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204394.728471:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204394.728471:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204394.728478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204394.728481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204394.728484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204394.920479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204394.920487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204394.920494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204394.920496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204394.920499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204394.920501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204394.920505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204394.920506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204394.920573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204394.920576:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204394.920579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204395.240497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204395.240503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204395.240505:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204395.240511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204395.240514:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204395.240520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204395.240523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204395.240526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204395.240529:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204395.688492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204395.688499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204395.688501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204395.688506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204395.688509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204395.688512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204395.688514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204395.688516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204395.688518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204395.752480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204395.752480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204395.752487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204395.752488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204395.752491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204395.752492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204395.752500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204395.752500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204395.752505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204395.752507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204395.752509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204395.752512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204395.752513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204395.752516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204395.752519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204395.752520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204395.752524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204395.752528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204395.944501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204395.944506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204395.944510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204395.944511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204395.944513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204395.944514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204395.944516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204395.944517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204395.944531:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204395.944533:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204395.944535:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204396.264470:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300523009, at 4300523520 00000100:00000010:1.0:1788204396.264483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000040:0.0:1788204396.264498:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300523009, at 4300523520 00000100:00000010:0.0:1788204396.264508:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00000001:1.0:1788204396.712482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204396.712489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204396.712490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204396.712494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204396.712497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204396.712500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204396.712502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204396.712507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204396.712510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204396.776496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204396.776503:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204396.776506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204396.776512:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204396.776519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204396.776526:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204396.776539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204396.776544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204396.776551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.776791:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204396.776796:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204396.776799:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204396.776808:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.776823:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204396.776828:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.776835:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204396.776840:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204396.776847:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204396.776858:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205858432:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204396.776868:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204396.776874:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204396.776880:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.776886:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204396.776893:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204396.776900:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204396.776911:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204396.776917:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204396.776927:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204396.776934:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205858432, portal 25 00000100:00000001:0.0:1788204396.776939:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204396.776942:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788204396.776951:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.776980:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204396.776986:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204396.776993:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204396.777000:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205858432, offset 0 00000400:00000200:0.0:1788204396.777015:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204396.777032:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.777043:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204396.777054:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204396.777066:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204396.777074:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204396.777084:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204396.777107:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58562:58562:256:4294967295] 192.168.201.151@tcp LPNI seq info [58542:58542:8:4294967295] 00000400:00000200:0.0:1788204396.777122:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.777144:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204396.777160:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.777171:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204396.777186:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204396.777195:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204396.777210:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204396.777221:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788204396.777238:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.777247:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204396.777258:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204396.777272:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204396.777282:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204396.777291:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204396.777301:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204396.777313:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777340:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.777351:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777420:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.777428:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204396.777441:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204396.777447:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204396.777451:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300528130 00000100:00000010:0.0:1788204396.777458:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204396.777466:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204396.777473:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300528130 00000100:00000010:0.0:1788204396.777477:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9840. 00000100:00000001:0.0:1788204396.777481:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204396.777485:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204396.777489:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.777492:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204396.777497:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204396/real 1788204396] req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204396.777509:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204396.777511:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204396.777514:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204396.777517:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204396.777520:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204396.777523:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204396.777528:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204396.777548:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777556:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204396.777558:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777561:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.777563:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204396.777569:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777576:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204396.777581:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777590:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204396.777593:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777595:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.777596:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777598:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204396.777600:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204396.777603:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204396.777607:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204396.777610:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204396.777614:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204396.777623:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205858432:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204396.777627:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777629:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.777633:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205858432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777639:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204396.777642:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204396.777644:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204396.777648:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204396.777650:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204396.777652:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788204396.777654:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204396.777657:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:0.0:1788204396.777660:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204396.777662:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204396.777664:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788204396.777667:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204396.777668:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777671:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777678:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204396.777679:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.777681:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204396.777683:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204396.777684:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204396.777688:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777695:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204396.777696:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.777699:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204396.777701:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204396.777703:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204396.777706:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858560:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204396.777710:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204396.777712:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204396.777714:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.777716:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204396.777720:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204396.777722:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204396.777726:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204396.777729:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204396.777733:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204396.777736:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205858560, portal 10 00000100:00000001:0.0:1788204396.777738:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204396.777740:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788204396.777747:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777755:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204396.777757:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204396.777760:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204396.777763:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205858560, offset 0 00000400:00000200:0.0:1788204396.777767:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204396.777771:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.777776:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204396.777779:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204396.777781:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204396.777784:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204396.777787:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204396.777790:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58563:58563:256:4294967295] 192.168.201.151@tcp LPNI seq info [58543:58543:8:4294967295] 00000400:00000200:0.0:1788204396.777795:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.777801:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204396.777806:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.777809:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204396.777811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204396.777813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204396.777817:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204396.777819:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788204396.777821:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.777823:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204396.777826:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204396.777828:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204396.777830:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204396.777831:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204396.777833:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204396.777836:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777842:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.777844:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777853:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.777854:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204396.777856:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204396.777859:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204396.777861:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204396.777863:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300524187 00000100:00000001:0.0:1788204396.777865:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.777866:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300524187 00000100:00000001:0.0:1788204396.777868:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.777869:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204396.777870:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.777872:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204396.777874:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204396/real 1788204396] req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204396.777879:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204396.777880:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204396.777881:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204396.777883:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204396.777884:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204396.777885:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204396.777887:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204396.777892:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777896:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204396.777897:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777898:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.777899:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204396.777901:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777906:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204396.777909:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777913:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204396.777914:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777915:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.777916:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.777917:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204396.777919:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204396.777921:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204396.777923:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204396.777924:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204396.777926:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204396.777931:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858560:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204396.777934:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204396.777936:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777940:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204396.777941:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.777942:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204396.777944:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204396.777945:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204396.777947:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204396.777948:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204396.777950:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204396.777951:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.777952:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204396.777955:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788204396.777957:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204396.777959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204396.777960:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204396.777963:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204396.777965:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205858688, portal 4 00000100:00000001:0.0:1788204396.777966:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204396.777967:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788204396.777988:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.777995:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204396.777998:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204396.778000:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204396.778003:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205858688, offset 0 00000400:00000200:0.0:1788204396.778006:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204396.778009:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.778012:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204396.778014:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204396.778016:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204396.778018:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204396.778019:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204396.778022:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58564:58564:256:4294967295] 192.168.201.151@tcp LPNI seq info [58544:58544:8:4294967295] 00000400:00000200:0.0:1788204396.778025:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.778029:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204396.778032:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.778034:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204396.778036:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204396.778038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204396.778041:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204396.778043:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788204396.778045:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.778047:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204396.778049:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204396.778052:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204396.778054:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204396.778055:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204396.778057:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204396.778059:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778065:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.778067:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778072:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.778073:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204396.778075:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204396.778077:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204396.778079:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204396.778080:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300524073 00000100:00000001:0.0:1788204396.778082:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.778084:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300524073 00000100:00000001:0.0:1788204396.778085:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.778086:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204396.778088:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.778089:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204396.778091:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204396/real 1788204396] req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204396.778096:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204396.778097:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204396.778098:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204396.778099:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204396.778100:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204396.778102:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204396.778104:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204396.778109:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778113:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204396.778114:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778115:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.778116:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204396.778118:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778122:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204396.778124:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778128:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204396.778130:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778131:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.778132:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778133:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204396.778134:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204396.778136:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204396.778138:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204396.778139:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204396.778142:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204396.778147:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204396.778149:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204396.778151:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778156:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204396.778157:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.778158:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204396.778160:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204396.778161:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204396.778163:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204396.778165:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204396.778166:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204396.778167:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204396.778168:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204396.778170:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788204396.778171:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204396.778173:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204396.778174:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204396.778176:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204396.778178:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205858816, portal 4 00000100:00000001:0.0:1788204396.778180:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204396.778181:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788204396.778187:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778194:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204396.778196:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204396.778198:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204396.778200:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205858816, offset 0 00000400:00000200:0.0:1788204396.778204:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204396.778208:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.778212:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204396.778215:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204396.778218:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204396.778220:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204396.778223:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204396.778226:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58565:58565:256:4294967295] 192.168.201.151@tcp LPNI seq info [58545:58545:8:4294967295] 00000400:00000200:0.0:1788204396.778231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204396.778237:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204396.778241:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.778244:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204396.778247:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204396.778248:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788204396.778251:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204396.778253:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788204396.778256:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204396.778257:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204396.778260:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204396.778262:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204396.778264:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204396.778265:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204396.778267:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204396.778270:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778275:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.778277:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778281:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.778282:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204396.778284:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204396.778286:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204396.778288:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204396.778290:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300524079 00000100:00000001:0.0:1788204396.778291:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.778293:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300524079 00000100:00000001:0.0:1788204396.778294:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300528130 00000100:00000001:0.0:1788204396.778295:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204396.778297:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.778298:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204396.778301:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204396/real 1788204396] req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204396.778306:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204396.778306:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204396.778307:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204396.778309:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204396.778310:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204396.778311:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204396.778313:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204396.778318:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778322:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204396.778323:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778324:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.778325:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204396.778327:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778331:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204396.778333:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778337:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204396.778339:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778340:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204396.778341:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778342:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204396.778361:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204396.778379:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204396.778382:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204396.778384:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204396.778387:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204396.778393:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205858816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204396.778395:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778397:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.778399:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205858560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778403:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204396.778405:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204396.778406:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204396.778408:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204396.778409:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204396.778410:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788204396.778412:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204396.778413:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788204396.778415:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204396.778416:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204396.778417:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788204396.778418:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204396.778419:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778420:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.778422:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205858688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778426:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204396.778427:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204396.778428:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788204396.778429:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204396.778430:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204396.778431:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788204396.778432:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204396.778433:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:0.0:1788204396.778435:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204396.778436:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204396.778437:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788204396.778438:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204396.778439:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778440:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204396.778443:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205858816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204396.778448:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204396.778449:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204396.778450:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788204396.778453:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204396.778454:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204396.778456:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204396.778458:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204396.778460:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788204396.778462:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204396.778463:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204396.778465:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788204396.778467:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204396.778468:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778470:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204396.778474:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204396.778476:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204396.778478:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204396.778479:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204396.968489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204396.968495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204396.968502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204396.968504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204396.968508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204396.968509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204396.968513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204396.968515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204396.968579:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204396.968583:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204396.968587:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204397.736456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204397.736464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204397.736466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204397.736472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204397.736476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204397.736480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204397.736482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204397.736486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204397.736489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204397.800490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204397.800500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204397.800503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204397.800511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204397.800518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204397.800525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204397.800532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204397.800536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204397.800540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204397.992533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204397.992540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204397.992545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204397.992547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204397.992550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204397.992551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204397.992554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204397.992555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204397.992570:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204397.992572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204397.992574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204398.760462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204398.760470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204398.760472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204398.760479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204398.760482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204398.760487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204398.760490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204398.760494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204398.760497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204398.824475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204398.824487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204398.824492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204398.824498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204398.824504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204398.824510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204398.824513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204398.824516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204398.824519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204399.016494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204399.016505:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204399.016507:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204399.016511:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016514:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204399.016526:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016528:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016531:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204399.016537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204399.016540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204399.016542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204399.016603:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204399.016607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204399.016609:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204399.784490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204399.784497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204399.784500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204399.784506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204399.784509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204399.784513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204399.784516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204399.784519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204399.784522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204399.848476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204399.848485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204399.848489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204399.848499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204399.848518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204399.848523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204399.848534:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204399.848538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204399.848541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204400.040530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204400.040538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204400.040544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204400.040545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204400.040549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204400.040550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204400.040554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204400.040555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204400.040573:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204400.040575:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204400.040577:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204400.808491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204400.808499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204400.808501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204400.808507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204400.808511:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204400.808515:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204400.808518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204400.808521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204400.808524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204400.872499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204400.872509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204400.872514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204400.872523:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204400.872531:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204400.872538:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204400.872545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204400.872550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204400.872555:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204401.064574:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204401.064576:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204401.064578:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064582:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204401.064583:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064585:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064587:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064589:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204401.064591:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064593:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204401.064594:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204401.064595:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204401.064596:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064597:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204401.064598:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064599:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064600:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064602:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204401.064603:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204401.064604:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204401.064605:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064606:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204401.064607:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064608:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204401.064609:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204401.064682:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204401.064686:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204401.064689:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204401.384437:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300528130, at 4300528640 00000100:00000010:1.0:1788204401.384458:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9840. 00000100:00000040:0.0:1788204401.384469:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300528130, at 4300528640 00000100:00000010:0.0:1788204401.384481:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000001:1.0:1788204401.832455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204401.832461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204401.832463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204401.832467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204401.832470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204401.832473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204401.832475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204401.832476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204401.832478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204401.896460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204401.896467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204401.896469:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204401.896473:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204401.896478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204401.896484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204401.896490:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204401.896496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204401.896505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.896743:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204401.896750:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204401.896754:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204401.896766:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.896793:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204401.896799:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.896805:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204401.896812:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204401.896818:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204401.896829:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205858944:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204401.896841:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204401.896849:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204401.896855:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.896873:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204401.896914:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204401.896921:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204401.896934:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204401.896944:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204401.896957:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204401.896969:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205858944, portal 25 00000100:00000001:0.0:1788204401.896978:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204401.896982:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300822400 : -113511408729216 : ffff98c310ce0380) 00000100:00000040:0.0:1788204401.896997:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897014:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204401.897021:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204401.897026:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204401.897031:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205858944, offset 0 00000400:00000200:0.0:1788204401.897041:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204401.897054:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.897066:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204401.897075:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204401.897084:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204401.897091:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204401.897099:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204401.897108:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58566:58566:256:4294967295] 192.168.201.151@tcp LPNI seq info [58546:58546:8:4294967295] 00000400:00000200:0.0:1788204401.897123:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.897141:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204401.897157:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.897167:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204401.897179:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204401.897185:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204401.897208:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204401.897213:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788204401.897223:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.897229:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204401.897240:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204401.897254:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204401.897264:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204401.897270:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204401.897278:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204401.897287:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897303:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.897311:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897328:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.897332:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204401.897338:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204401.897346:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204401.897350:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300533249 00000100:00000010:0.0:1788204401.897358:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204401.897411:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204401.897422:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300533250 00000100:00000010:0.0:1788204401.897427:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff94e0. 00000100:00000001:0.0:1788204401.897431:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204401.897435:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204401.897438:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.897441:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204401.897446:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204401/real 1788204401] req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204401.897455:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204401.897457:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204401.897459:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204401.897470:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204401.897472:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204401.897474:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204401.897478:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204401.897487:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897494:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204401.897496:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897498:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.897500:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204401.897504:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897511:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204401.897516:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897523:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204401.897527:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897528:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.897530:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897532:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204401.897534:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204401.897537:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204401.897541:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204401.897544:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204401.897549:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204401.897557:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205858944:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204401.897561:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897564:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.897567:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0380 x1875062205858944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897573:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204401.897576:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204401.897578:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204401.897582:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204401.897583:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204401.897585:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788204401.897587:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204401.897590:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:0.0:1788204401.897593:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204401.897595:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204401.897597:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0380. 00000100:00000001:0.0:1788204401.897600:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204401.897601:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897611:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897617:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204401.897619:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.897621:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204401.897623:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204401.897625:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204401.897628:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897636:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204401.897637:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.897639:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204401.897641:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204401.897643:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204401.897646:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859072:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204401.897649:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204401.897651:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204401.897653:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.897654:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204401.897657:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204401.897659:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204401.897662:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204401.897664:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204401.897667:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204401.897670:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859072, portal 10 00000100:00000001:0.0:1788204401.897672:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204401.897673:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:0.0:1788204401.897679:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897685:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204401.897688:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204401.897690:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204401.897693:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205859072, offset 0 00000400:00000200:0.0:1788204401.897697:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204401.897702:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.897706:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204401.897709:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204401.897712:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204401.897715:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204401.897717:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204401.897721:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58567:58567:256:4294967295] 192.168.201.151@tcp LPNI seq info [58547:58547:8:4294967295] 00000400:00000200:0.0:1788204401.897725:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.897732:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204401.897736:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.897740:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204401.897744:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204401.897747:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204401.897751:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204401.897754:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788204401.897758:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.897760:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204401.897764:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204401.897767:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204401.897770:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204401.897772:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204401.897774:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204401.897778:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897786:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.897789:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897795:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.897797:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204401.897799:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204401.897803:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204401.897806:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204401.897808:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300529307 00000100:00000001:0.0:1788204401.897811:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300533249 00000100:00000001:0.0:1788204401.897813:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300529307 00000100:00000001:0.0:1788204401.897815:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300533250 00000100:00000001:0.0:1788204401.897817:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204401.897820:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.897822:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204401.897826:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204401/real 1788204401] req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204401.897834:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204401.897835:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204401.897837:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204401.897839:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204401.897841:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204401.897842:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204401.897845:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204401.897854:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897860:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204401.897861:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897863:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.897865:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204401.897868:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897875:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204401.897892:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897900:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204401.897903:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897905:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.897906:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.897908:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204401.897910:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204401.897913:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204401.897916:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204401.897919:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204401.897923:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204401.897930:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859072:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204401.897934:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204401.897937:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897943:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204401.897944:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.897946:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204401.897948:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204401.897950:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204401.897952:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204401.897955:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204401.897957:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204401.897959:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.897961:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204401.897964:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204401.897966:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204401.897969:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204401.897971:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204401.897975:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204401.897978:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859200, portal 4 00000100:00000001:0.0:1788204401.897980:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204401.897981:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:0.0:1788204401.897987:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.897994:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204401.897996:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204401.897999:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204401.898002:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205859200, offset 0 00000400:00000200:0.0:1788204401.898006:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204401.898010:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.898015:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204401.898018:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204401.898021:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204401.898023:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204401.898026:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204401.898029:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58568:58568:256:4294967295] 192.168.201.151@tcp LPNI seq info [58548:58548:8:4294967295] 00000400:00000200:0.0:1788204401.898034:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.898041:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204401.898046:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.898049:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204401.898054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204401.898057:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204401.898061:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204401.898064:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788204401.898068:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.898070:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204401.898073:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204401.898077:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204401.898080:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204401.898082:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204401.898085:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204401.898089:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898098:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.898102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898110:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.898112:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204401.898115:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204401.898118:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204401.898122:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204401.898124:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300529193 00000100:00000001:0.0:1788204401.898127:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300533249 00000100:00000001:0.0:1788204401.898129:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300529193 00000100:00000001:0.0:1788204401.898131:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300533250 00000100:00000001:0.0:1788204401.898133:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204401.898135:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.898137:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204401.898141:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204401/real 1788204401] req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204401.898150:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204401.898152:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204401.898154:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204401.898156:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204401.898158:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204401.898160:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204401.898163:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204401.898172:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898179:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204401.898181:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898183:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.898184:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204401.898189:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898196:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204401.898200:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898208:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204401.898210:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898212:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.898213:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898215:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204401.898218:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204401.898221:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204401.898224:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204401.898226:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204401.898230:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204401.898238:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204401.898242:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204401.898245:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898251:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204401.898253:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.898255:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204401.898257:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204401.898259:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204401.898261:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204401.898264:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204401.898266:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204401.898268:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204401.898270:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204401.898273:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788204401.898276:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204401.898279:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204401.898282:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204401.898285:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204401.898288:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859328, portal 4 00000100:00000001:0.0:1788204401.898290:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204401.898291:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:0.0:1788204401.898297:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898303:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204401.898305:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204401.898308:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204401.898310:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205859328, offset 0 00000400:00000200:0.0:1788204401.898314:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204401.898318:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.898321:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204401.898324:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204401.898327:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204401.898329:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204401.898332:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204401.898335:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58569:58569:256:4294967295] 192.168.201.151@tcp LPNI seq info [58549:58549:8:4294967295] 00000400:00000200:0.0:1788204401.898339:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204401.898345:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204401.898350:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.898353:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204401.898357:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204401.898360:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788204401.898389:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204401.898392:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788204401.898396:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204401.898398:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204401.898403:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204401.898408:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204401.898412:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204401.898414:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204401.898417:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204401.898421:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898429:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.898433:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898439:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.898441:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204401.898444:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204401.898446:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204401.898450:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204401.898452:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300529200 00000100:00000001:0.0:1788204401.898454:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300533249 00000100:00000001:0.0:1788204401.898456:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300529200 00000100:00000001:0.0:1788204401.898459:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300533250 00000100:00000001:0.0:1788204401.898461:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204401.898463:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.898465:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204401.898469:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204401/real 1788204401] req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204401.898477:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204401.898479:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204401.898480:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204401.898483:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204401.898485:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204401.898487:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204401.898490:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204401.898499:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898512:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204401.898514:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898516:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.898518:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204401.898522:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898529:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204401.898533:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898540:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204401.898542:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898543:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204401.898545:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898546:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204401.898548:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204401.898551:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204401.898554:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204401.898557:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204401.898561:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204401.898569:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204401.898572:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898574:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.898578:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205859072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898585:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204401.898586:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204401.898589:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204401.898591:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204401.898593:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204401.898595:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788204401.898597:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204401.898599:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:0.0:1788204401.898601:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204401.898603:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204401.898604:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:0.0:1788204401.898607:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204401.898608:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898610:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.898613:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205859200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898620:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204401.898622:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204401.898623:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204401.898626:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204401.898627:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204401.898629:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788204401.898631:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204401.898632:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:0.0:1788204401.898635:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204401.898636:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204401.898637:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:0.0:1788204401.898640:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204401.898642:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898643:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204401.898647:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205859328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204401.898654:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204401.898655:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204401.898657:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788204401.898659:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204401.898660:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204401.898661:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204401.898664:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204401.898666:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:0.0:1788204401.898668:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204401.898669:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204401.898671:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:0.0:1788204401.898673:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204401.898674:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898676:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204401.898679:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204401.898681:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204401.898682:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204401.898684:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204402.088517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204402.088525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204402.088531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204402.088532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204402.088536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204402.088537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204402.088540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204402.088542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204402.088559:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204402.088562:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204402.088565:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204402.856500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204402.856507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204402.856509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204402.856513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204402.856517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204402.856520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204402.856522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204402.856524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204402.856527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204402.920484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204402.920492:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204402.920495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204402.920502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204402.920508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204402.920514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204402.920520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204402.920522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204402.920524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204403.112492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204403.112500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204403.112508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204403.112510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204403.112516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204403.112518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204403.112523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204403.112531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204403.112565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204403.112569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204403.112573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204403.880477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204403.880484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204403.880486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204403.880490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204403.880492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204403.880495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204403.880497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204403.880499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204403.880500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204403.944469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204403.944474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204403.944476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204403.944481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204403.944484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204403.944487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204403.944489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204403.944492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204403.944497:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204404.136492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204404.136497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204404.136501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204404.136502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204404.136504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204404.136505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204404.136507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204404.136508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204404.136520:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204404.136522:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204404.136523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204404.904482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204404.904490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204404.904492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204404.904497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204404.904501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204404.904505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204404.904508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204404.904510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204404.904513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204404.968469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204404.968476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204404.968480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204404.968487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204404.968493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204404.968498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204404.968509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204404.968512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204404.968514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204405.160479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204405.160485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204405.160491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204405.160493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204405.160496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204405.160497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204405.160500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204405.160501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204405.160558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204405.160560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204405.160562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204405.928480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204405.928485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204405.928487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204405.928491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204405.928493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204405.928496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204405.928498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204405.928500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204405.928502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204405.992489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204405.992495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204405.992498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204405.992504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204405.992508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204405.992513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204405.992516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204405.992519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204405.992522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204406.184510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204406.184518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204406.184524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204406.184526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204406.184529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204406.184530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204406.184534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204406.184535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204406.184555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204406.184557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204406.184560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204406.504458:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300533249, at 4300533760 00000100:00000040:1.0:1788204406.504458:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300533250, at 4300533760 00000100:00000010:0.0:1788204406.504470:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000010:1.0:1788204406.504472:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff94e0. 00000100:00000001:1.0:1788204406.952512:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204406.952519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204406.952520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204406.952525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204406.952528:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204406.952531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204406.952534:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204406.952536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204406.952539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204407.016481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204407.016487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204407.016489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204407.016494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204407.016499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204407.016502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204407.016505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204407.016507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204407.016510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.016715:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204407.016721:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204407.016725:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204407.016735:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.016754:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204407.016758:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.016763:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204407.016768:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204407.016773:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204407.016787:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205859456:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204407.016810:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204407.016816:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204407.016821:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.016825:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204407.016834:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204407.016838:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204407.016850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204407.016857:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204407.016867:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204407.016877:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859456, portal 25 00000100:00000001:0.0:1788204407.016882:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204407.016887:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:0.0:1788204407.016899:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.016918:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204407.016927:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204407.016932:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204407.016938:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205859456, offset 0 00000400:00000200:0.0:1788204407.016950:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204407.016962:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.016973:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204407.016982:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204407.016991:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204407.016999:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204407.017007:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017017:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58570:58570:256:4294967295] 192.168.201.151@tcp LPNI seq info [58550:58550:8:4294967295] 00000400:00000200:0.0:1788204407.017028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017049:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204407.017061:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017069:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204407.017079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204407.017083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204407.017094:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204407.017099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204407.017119:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017123:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204407.017133:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204407.017142:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204407.017151:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204407.017156:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204407.017163:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204407.017171:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017185:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017190:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017196:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017197:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204407.017202:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204407.017205:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204407.017207:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300538369 00000100:00000010:0.0:1788204407.017211:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204407.017215:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204407.017220:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300538369 00000100:00000010:0.0:1788204407.017222:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff94e0. 00000100:00000001:0.0:1788204407.017224:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204407.017226:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204407.017228:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017229:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204407.017233:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204407.017238:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204407.017240:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204407.017241:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204407.017243:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204407.017244:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204407.017245:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204407.017248:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204407.017256:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017260:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204407.017261:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017263:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017264:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204407.017266:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017270:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204407.017272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017276:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204407.017278:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017279:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017279:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017281:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204407.017282:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204407.017283:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.017286:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204407.017287:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204407.017290:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.017295:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205859456:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204407.017297:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017298:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017300:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205859456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017307:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204407.017309:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017310:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204407.017312:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204407.017313:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204407.017314:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788204407.017315:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204407.017317:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:0.0:1788204407.017320:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204407.017321:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204407.017322:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:0.0:1788204407.017324:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204407.017324:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017326:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017330:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204407.017331:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017332:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204407.017333:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204407.017334:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204407.017336:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017340:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204407.017341:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017342:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204407.017343:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204407.017344:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204407.017346:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859584:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204407.017348:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204407.017349:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204407.017350:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017351:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017353:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204407.017354:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204407.017356:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204407.017358:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204407.017360:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204407.017362:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859584, portal 10 00000100:00000001:0.0:1788204407.017391:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204407.017392:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:0.0:1788204407.017397:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017401:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204407.017402:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204407.017404:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204407.017406:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205859584, offset 0 00000400:00000200:0.0:1788204407.017408:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017410:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017420:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204407.017422:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204407.017424:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204407.017425:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204407.017427:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017428:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58571:58571:256:4294967295] 192.168.201.151@tcp LPNI seq info [58551:58551:8:4294967295] 00000400:00000200:0.0:1788204407.017431:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017435:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204407.017438:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017440:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204407.017443:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204407.017444:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204407.017447:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204407.017460:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204407.017464:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017465:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204407.017468:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204407.017471:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204407.017475:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204407.017477:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204407.017479:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204407.017483:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017490:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017493:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017499:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017501:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204407.017504:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204407.017507:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204407.017511:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204407.017512:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300534427 00000100:00000001:0.0:1788204407.017514:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017515:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300534427 00000100:00000001:0.0:1788204407.017516:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017518:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204407.017519:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017520:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204407.017522:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204407.017527:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204407.017528:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204407.017529:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204407.017530:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204407.017531:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204407.017532:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204407.017534:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204407.017539:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017546:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204407.017547:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017548:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017549:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204407.017551:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017554:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204407.017556:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017560:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204407.017561:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017562:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017563:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017564:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204407.017565:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204407.017567:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.017569:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204407.017570:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204407.017572:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.017576:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859584:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204407.017578:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204407.017580:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017584:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204407.017584:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017586:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204407.017587:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204407.017588:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204407.017590:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204407.017592:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204407.017593:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204407.017594:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017595:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017597:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204407.017598:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204407.017600:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204407.017602:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204407.017603:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204407.017605:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859712, portal 4 00000100:00000001:0.0:1788204407.017606:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204407.017607:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:0.0:1788204407.017610:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017614:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204407.017615:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204407.017617:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204407.017618:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205859712, offset 0 00000400:00000200:0.0:1788204407.017620:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017622:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017624:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204407.017626:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204407.017627:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204407.017629:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204407.017630:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017632:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58572:58572:256:4294967295] 192.168.201.151@tcp LPNI seq info [58552:58552:8:4294967295] 00000400:00000200:0.0:1788204407.017635:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017638:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204407.017642:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017643:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204407.017646:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204407.017647:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204407.017650:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204407.017651:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204407.017653:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017655:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204407.017657:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204407.017659:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204407.017660:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204407.017661:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204407.017663:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204407.017665:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017669:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017671:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017675:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017676:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204407.017677:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204407.017679:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204407.017680:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204407.017681:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300534313 00000100:00000001:0.0:1788204407.017683:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017684:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300534313 00000100:00000001:0.0:1788204407.017685:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017686:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204407.017687:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017688:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204407.017690:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204407.017695:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204407.017695:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204407.017696:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204407.017698:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204407.017698:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204407.017699:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204407.017701:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204407.017705:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017708:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204407.017709:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017710:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017711:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204407.017712:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017716:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204407.017718:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017721:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204407.017722:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017723:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017724:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017725:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204407.017726:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204407.017727:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.017729:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204407.017730:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204407.017733:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.017737:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204407.017739:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204407.017741:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017744:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204407.017745:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017746:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204407.017747:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204407.017748:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204407.017749:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204407.017750:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204407.017751:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204407.017752:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204407.017753:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017755:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788204407.017756:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204407.017757:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204407.017758:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204407.017759:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204407.017761:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859840, portal 4 00000100:00000001:0.0:1788204407.017762:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204407.017763:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000040:0.0:1788204407.017766:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017770:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204407.017771:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204407.017772:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204407.017773:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205859840, offset 0 00000400:00000200:0.0:1788204407.017776:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017779:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204407.017781:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204407.017782:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204407.017784:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204407.017785:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204407.017786:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58573:58573:256:4294967295] 192.168.201.151@tcp LPNI seq info [58553:58553:8:4294967295] 00000400:00000200:0.0:1788204407.017789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204407.017808:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204407.017811:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017813:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204407.017815:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204407.017817:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204407.017819:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204407.017820:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204407.017823:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204407.017824:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204407.017826:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204407.017828:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204407.017829:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204407.017830:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204407.017832:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204407.017834:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017838:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017840:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204432 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017844:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017845:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204407.017846:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204407.017848:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204407.017850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204407.017851:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300534319 00000100:00000001:0.0:1788204407.017853:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017854:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300534319 00000100:00000001:0.0:1788204407.017855:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300538369 00000100:00000001:0.0:1788204407.017856:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204407.017857:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017858:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204407.017860:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204407.017864:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204407.017865:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204407.017866:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204407.017867:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204407.017868:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204407.017869:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204407.017871:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204407.017876:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017879:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204407.017880:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017881:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017881:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204407.017883:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017887:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204407.017889:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017892:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204407.017894:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017895:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204407.017895:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017896:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204407.017898:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204407.017899:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.017901:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204407.017902:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204407.017904:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.017908:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205859840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204407.017909:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017910:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017912:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205859584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017915:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204407.017916:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017917:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204407.017918:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204407.017919:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204407.017920:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788204407.017921:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204407.017923:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:0.0:1788204407.017924:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204407.017925:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204407.017926:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:0.0:1788204407.017928:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204407.017928:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017929:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017931:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205859712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017938:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204407.017939:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017940:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788204407.017942:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204407.017942:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204407.017943:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788204407.017944:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204407.017944:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:0.0:1788204407.017945:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204407.017946:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204407.017947:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:0.0:1788204407.017948:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204407.017948:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017949:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204407.017951:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205859840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204407.017954:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204407.017954:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204407.017955:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788204407.017956:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204407.017957:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204407.017957:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204407.017958:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204407.017959:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:0.0:1788204407.017960:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204407.017960:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204407.017961:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:0.0:1788204407.017962:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204407.017963:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017964:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204407.017966:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204407.017967:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204407.017968:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204407.017969:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204407.208461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204407.208466:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204407.208470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204407.208471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204407.208473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204407.208474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204407.208480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204407.208481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204407.208545:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204407.208549:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204407.208552:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204407.976460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204407.976465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204407.976467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204407.976471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204407.976473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204407.976476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204407.976478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204407.976480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204407.976481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204408.040472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204408.040480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204408.040484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204408.040489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204408.040494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204408.040498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204408.040506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204408.040509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204408.040512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204408.232578:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232583:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204408.232586:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204408.232588:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204408.232591:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232594:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204408.232596:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232598:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232600:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232603:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204408.232604:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204408.232608:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204408.232609:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204408.232624:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204408.232627:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204408.232629:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204409.000461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204409.000468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204409.000470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204409.000475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204409.000479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204409.000482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204409.000484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204409.000486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204409.000489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204409.064489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204409.064497:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204409.064500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204409.064508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204409.064512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204409.064516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204409.064519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204409.064523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204409.064525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204409.256473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204409.256479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204409.256483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204409.256484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204409.256487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204409.256488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204409.256490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204409.256491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204409.256541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204409.256544:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204409.256546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204410.024482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204410.024487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204410.024488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204410.024492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204410.024495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204410.024498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204410.024500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204410.024501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204410.024503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204410.088460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204410.088467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204410.088468:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204410.088473:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204410.088476:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204410.088479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204410.088481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204410.088483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204410.088485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204410.280506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204410.280510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204410.280513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204410.280514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204410.280516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204410.280517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204410.280519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204410.280519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204410.280531:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204410.280533:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204410.280535:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204411.048445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204411.048453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204411.048455:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204411.048461:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204411.048465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204411.048468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204411.048471:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204411.048473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204411.048476:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204411.112456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204411.112463:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204411.112466:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204411.112473:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204411.112478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204411.112482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204411.112491:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204411.112493:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204411.112496:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204411.304472:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204411.304474:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204411.304479:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304484:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204411.304487:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304489:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304491:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204411.304497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204411.304502:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204411.304503:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204411.304504:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304506:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204411.304507:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304509:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304512:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204411.304516:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204411.304518:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204411.304519:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304521:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204411.304522:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304523:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204411.304525:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204411.304570:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204411.304573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204411.304575:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204411.624462:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300538369, at 4300538880 00000100:00000040:0.0:1788204411.624463:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300538369, at 4300538880 00000100:00000010:1.0:1788204411.624475:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff94e0. 00000100:00000010:0.0:1788204411.624477:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000001:1.0:1788204412.072489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204412.072496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204412.072497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204412.072503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204412.072506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204412.072509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204412.072512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204412.072514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204412.072518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204412.136433:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204412.136439:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204412.136442:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204412.136448:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204412.136452:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204412.136457:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204412.136460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204412.136463:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204412.136471:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.136695:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204412.136699:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204412.136713:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204412.136720:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.136732:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204412.136734:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.136738:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204412.136741:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204412.136743:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204412.136749:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205859968:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204412.136756:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204412.136759:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204412.136762:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.136766:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204412.136771:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204412.136775:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204412.136785:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204412.136790:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204412.136795:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204412.136801:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205859968, portal 25 00000100:00000001:0.0:1788204412.136806:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204412.136809:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:0.0:1788204412.136816:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.136824:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204412.136827:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204412.136832:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204412.136835:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205859968, offset 0 00000400:00000200:0.0:1788204412.136841:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204412.136849:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.136854:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204412.136860:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204412.136865:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204412.136872:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204412.136877:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204412.136886:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58574:58574:256:4294967295] 192.168.201.151@tcp LPNI seq info [58554:58554:8:4294967295] 00000400:00000200:0.0:1788204412.136898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.136916:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204412.136929:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.136935:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204412.136944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204412.136960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788204412.136976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204412.136983:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788204412.136993:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.136996:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204412.137006:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204412.137014:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204412.137025:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204412.137030:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204412.137037:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204412.137044:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137061:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137069:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137097:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137102:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204412.137108:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204412.137116:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204412.137127:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300543489 00000100:00000010:0.0:1788204412.137134:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9480. 00000100:00000001:0.0:1788204412.137140:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204412.137149:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300543489 00000100:00000010:0.0:1788204412.137155:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9840. 00000100:00000001:0.0:1788204412.137160:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204412.137165:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204412.137170:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137177:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204412.137186:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204412.137212:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204412.137215:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204412.137218:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204412.137222:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204412.137225:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204412.137229:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204412.137235:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204412.137251:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137266:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204412.137268:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137271:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137273:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204412.137278:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137295:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204412.137302:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137312:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204412.137316:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137317:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137318:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137320:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204412.137321:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204412.137323:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.137326:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204412.137328:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204412.137331:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.137335:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205859968:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204412.137339:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137340:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137342:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205859968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137345:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204412.137347:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137349:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204412.137351:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204412.137352:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204412.137353:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788204412.137354:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204412.137356:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:0.0:1788204412.137358:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204412.137360:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204412.137361:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:0.0:1788204412.137390:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204412.137391:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137393:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137398:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204412.137399:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137400:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204412.137401:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204412.137402:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204412.137405:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137409:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204412.137410:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137412:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204412.137413:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204412.137414:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204412.137416:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860096:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204412.137418:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204412.137419:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204412.137420:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137422:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137424:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204412.137425:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204412.137427:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204412.137429:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204412.137431:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204412.137433:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205860096, portal 10 00000100:00000001:0.0:1788204412.137434:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204412.137435:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:0.0:1788204412.137439:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137451:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204412.137454:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204412.137456:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204412.137458:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205860096, offset 0 00000400:00000200:0.0:1788204412.137460:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137463:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137465:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204412.137467:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204412.137469:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204412.137471:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204412.137472:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137474:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58575:58575:256:4294967295] 192.168.201.151@tcp LPNI seq info [58555:58555:8:4294967295] 00000400:00000200:0.0:1788204412.137476:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137480:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204412.137483:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137485:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204412.137487:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204412.137489:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788204412.137492:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204412.137494:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788204412.137496:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137497:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204412.137500:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204412.137502:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204412.137504:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204412.137505:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204412.137506:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204412.137509:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137513:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137515:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137519:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137520:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204412.137522:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204412.137524:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788204412.137526:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204412.137528:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300539547 00000100:00000001:0.0:1788204412.137529:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137531:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300539547 00000100:00000001:0.0:1788204412.137532:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137533:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204412.137534:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137535:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204412.137537:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204412.137541:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204412.137542:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204412.137543:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204412.137544:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204412.137545:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204412.137546:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204412.137548:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204412.137552:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137555:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204412.137556:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137557:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137558:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204412.137560:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137563:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204412.137565:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137570:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204412.137571:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137572:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137573:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137574:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204412.137575:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204412.137577:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.137580:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204412.137582:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204412.137585:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.137592:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860096:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788204412.137596:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204412.137599:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137605:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204412.137606:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137608:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204412.137610:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204412.137611:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204412.137614:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204412.137617:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204412.137619:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204412.137620:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137622:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137625:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788204412.137627:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204412.137630:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204412.137633:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204412.137636:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204412.137639:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205860224, portal 4 00000100:00000001:0.0:1788204412.137640:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204412.137641:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:0.0:1788204412.137646:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137649:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204412.137651:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204412.137652:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204412.137653:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205860224, offset 0 00000400:00000200:0.0:1788204412.137655:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137658:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137660:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204412.137661:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204412.137663:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204412.137665:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204412.137666:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137668:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58576:58576:256:4294967295] 192.168.201.151@tcp LPNI seq info [58556:58556:8:4294967295] 00000400:00000200:0.0:1788204412.137670:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137674:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204412.137677:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137679:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204412.137681:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204412.137683:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788204412.137686:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204412.137687:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788204412.137689:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137690:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204412.137693:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204412.137695:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204412.137696:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204412.137697:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204412.137699:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204412.137714:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137720:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137722:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137725:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137726:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204412.137729:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204412.137731:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788204412.137733:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204412.137734:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300539433 00000100:00000001:0.0:1788204412.137735:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137737:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300539433 00000100:00000001:0.0:1788204412.137738:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137739:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204412.137740:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137741:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204412.137743:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204412.137747:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204412.137748:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204412.137749:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204412.137750:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204412.137751:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204412.137752:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204412.137754:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204412.137759:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137766:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204412.137767:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137768:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137769:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204412.137771:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137774:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204412.137776:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137780:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204412.137781:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137782:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137782:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137783:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204412.137785:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204412.137786:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.137788:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204412.137789:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204412.137791:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.137795:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204412.137797:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204412.137799:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137802:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204412.137803:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137805:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204412.137806:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204412.137808:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204412.137809:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204412.137811:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204412.137812:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204412.137813:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204412.137814:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137816:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204412.137817:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204412.137819:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788204412.137820:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788204412.137822:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788204412.137823:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205860352, portal 4 00000100:00000001:0.0:1788204412.137825:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204412.137825:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:0.0:1788204412.137829:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137833:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204412.137834:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788204412.137835:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788204412.137837:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205860352, offset 0 00000400:00000200:0.0:1788204412.137838:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137840:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137842:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204412.137844:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204412.137845:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204412.137846:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204412.137848:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204412.137849:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58577:58577:256:4294967295] 192.168.201.151@tcp LPNI seq info [58557:58557:8:4294967295] 00000400:00000200:0.0:1788204412.137852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204412.137855:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788204412.137858:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137860:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788204412.137862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788204412.137864:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788204412.137866:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788204412.137868:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788204412.137870:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788204412.137871:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788204412.137873:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204412.137875:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204412.137877:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788204412.137877:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788204412.137879:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204412.137881:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137885:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137887:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137890:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137891:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204412.137892:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204412.137893:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788204412.137895:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204412.137896:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300539439 00000100:00000001:0.0:1788204412.137897:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137898:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300539439 00000100:00000001:0.0:1788204412.137899:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300543489 00000100:00000001:0.0:1788204412.137901:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204412.137902:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.137903:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204412.137905:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204412.137909:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204412.137910:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204412.137911:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788204412.137912:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788204412.137913:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788204412.137914:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204412.137915:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204412.137920:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137923:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204412.137923:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137924:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137925:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204412.137927:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137930:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204412.137932:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137935:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204412.137937:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137937:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204412.137938:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137939:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204412.137940:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204412.137942:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.137943:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204412.137945:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204412.137947:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.137952:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205860352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204412.137954:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137955:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137957:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205860096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137960:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204412.137961:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137962:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788204412.137963:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204412.137964:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204412.137965:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788204412.137966:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204412.137966:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:0.0:1788204412.137968:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204412.137969:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204412.137970:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:0.0:1788204412.137971:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204412.137972:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137973:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137974:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205860224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137977:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204412.137978:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137979:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788204412.137980:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204412.137981:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204412.137981:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788204412.137982:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204412.137983:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:0.0:1788204412.137984:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204412.137985:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204412.137986:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:0.0:1788204412.137987:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204412.137988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.137989:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204412.137990:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205860352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204412.137994:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204412.137994:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204412.137995:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788204412.137996:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204412.137996:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788204412.137997:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788204412.137998:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788204412.137999:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:0.0:1788204412.138000:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204412.138000:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204412.138001:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:0.0:1788204412.138002:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204412.138003:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.138005:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204412.138008:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204412.138009:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204412.138011:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204412.138012:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204412.328506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204412.328513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204412.328519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204412.328520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204412.328522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204412.328523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204412.328526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204412.328527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204412.328544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204412.328546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204412.328549:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204413.096466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204413.096472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204413.096474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204413.096479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204413.096482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204413.096486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204413.096488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204413.096491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204413.096493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204413.160473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204413.160479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204413.160482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204413.160488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204413.160493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204413.160496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204413.160505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204413.160507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204413.160509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204413.352470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204413.352477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204413.352484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204413.352486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204413.352490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204413.352491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204413.352495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204413.352496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204413.352553:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204413.352557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204413.352560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204414.120483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204414.120490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204414.120492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204414.120499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204414.120502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204414.120506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204414.120508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204414.120510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204414.120513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204414.184465:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204414.184472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204414.184474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204414.184479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204414.184484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204414.184487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204414.184492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204414.184496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204414.184501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204414.376488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204414.376494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204414.376497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204414.376498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204414.376500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204414.376501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204414.376503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204414.376503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204414.376517:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204414.376520:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204414.376523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204415.144473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204415.144480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204415.144483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204415.144488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204415.144492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204415.144496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204415.144499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204415.144502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204415.144505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204415.208474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204415.208481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204415.208485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204415.208495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204415.208501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204415.208504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204415.208514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204415.208517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204415.208520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204415.400438:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204415.400443:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204415.400446:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204415.400447:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204415.400449:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204415.400450:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204415.400452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204415.400452:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204415.400501:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204415.400504:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204415.400508:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204416.168508:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204416.168516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204416.168518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204416.168524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204416.168527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204416.168531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204416.168533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204416.168536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204416.168538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204416.232650:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204416.232653:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204416.232654:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204416.232657:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204416.232659:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204416.232661:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204416.232663:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204416.232665:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204416.232667:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204416.424508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204416.424514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204416.424518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204416.424519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204416.424521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204416.424522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204416.424524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204416.424525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204416.424539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204416.424540:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204416.424542:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204416.744462:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300543489, at 4300544000 00000100:00000040:1.0:1788204416.744462:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300543489, at 4300544000 00000100:00000010:0.0:1788204416.744481:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9480. 00000100:00000010:1.0:1788204416.744484:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9840. 00000100:00000001:1.0:1788204417.192478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204417.192486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204417.192488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204417.192493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204417.192497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204417.192500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204417.192502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204417.192505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204417.192507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204417.256482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204417.256483:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204417.256492:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204417.256493:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204417.256496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256497:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204417.256503:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204417.256503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204417.256509:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204417.256509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204417.256514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204417.256519:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5872 00000100:00000001:1.0:1788204417.256522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204417.256526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204417.256530:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204417.256531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204417.256537:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204417.256541:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204417.256543:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256547:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.256549:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256551:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204417.256556:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204417.256558:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204417.256562:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204417.256565:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204417.256568:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204417.256571:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204417.256574:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256576:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.256578:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256580:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204417.256584:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204417.256587:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256590:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204417.256594:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204417.256598:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204417.256600:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256602:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204417.256605:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204417.256634:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204417.256638:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204417.256642:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204417.256644:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256646:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204417.256648:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204417.256652:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204417.256661:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204417.256668:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204417.256676:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205860480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204417.256687:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204417.256692:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204417.256697:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205860480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204417.256722:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204417.256725:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204417.256728:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204417.256730:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204417.256731:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204417.256740:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204417.256743:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5872 00000400:00000200:0.0:1788204417.256747:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204417.256751:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204417.256753:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204417.256756:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256758:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.256761:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256764:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204417.256766:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204417.256768:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204417.256772:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204417.256774:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204417.256777:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204417.256780:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204417.256784:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256786:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.256787:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256790:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204417.256792:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204417.256796:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204417.256799:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204417.256805:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204417.256812:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204417.256817:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256822:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204417.256828:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204417.256835:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256841:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204417.256854:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204417.256862:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204417.256868:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204417.256875:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.256879:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204417.256882:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204417.256885:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204417.256893:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204417.256902:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204417.256915:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205860608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204417.256927:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204417.256932:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204417.256943:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205860608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204417.256972:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204417.256978:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204417.256987:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204417.256988:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204417.256991:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204417.257002:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204417.257008:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5872 00000400:00000200:0.0:1788204417.257017:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204417.257025:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204417.257028:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204417.257034:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257039:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.257047:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257055:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204417.257065:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204417.257071:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204417.257074:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204417.257076:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204417.257079:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204417.257084:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204417.257092:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257096:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.257097:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257099:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204417.257111:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204417.257116:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257121:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204417.257124:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204417.257129:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204417.257135:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257137:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204417.257140:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204417.257148:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257153:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204417.257158:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204417.257168:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204417.257177:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204417.257182:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257191:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204417.257195:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204417.257207:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204417.257220:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204417.257231:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204417.257242:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205860736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204417.257285:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1788204417.257286:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204417.257291:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204417.257293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1788204417.257300:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205860736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204417.257326:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204417.257331:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204417.257339:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204417.257341:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204417.257342:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204417.257346:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204417.257353:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5872 00000400:00000200:0.0:1788204417.257361:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204417.257391:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204417.257394:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204417.257397:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257399:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.257401:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257402:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204417.257404:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204417.257405:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204417.257407:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204417.257409:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204417.257416:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204417.257419:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204417.257422:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257428:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204417.257430:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257433:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204417.257436:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204417.257441:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204417.257445:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204417.257452:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204417.257456:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204417.257460:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257463:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204417.257466:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204417.257471:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257475:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204417.257478:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204417.257481:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204417.257486:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204417.257489:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204417.257490:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204417.257492:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204417.257495:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204417.257500:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204417.257507:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204417.257514:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205860864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204417.257522:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204417.257526:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204417.257532:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205860864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204417.257547:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204417.257554:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5882) 00000800:00000200:2.1:1788204417.257852:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204417.257854:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204417.258108:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204417.258110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204417.258292:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204417.258293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204417.448486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204417.448494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204417.448498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204417.448499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204417.448503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204417.448504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204417.448507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204417.448508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204417.448565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204417.448570:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204417.448574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204418.216506:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204418.216517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204418.216519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204418.216525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204418.216529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204418.216541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204418.216544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204418.216547:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204418.216549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204418.280503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204418.280511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204418.280514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204418.280521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204418.280526:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204418.280530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204418.280533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204418.280541:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204418.280544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204418.472532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204418.472539:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204418.472541:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204418.472545:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472547:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204418.472549:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472551:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472552:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204418.472556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204418.472558:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204418.472558:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204418.472573:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204418.472575:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204418.472578:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204419.240480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204419.240487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204419.240489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204419.240494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204419.240497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204419.240501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204419.240503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204419.240505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204419.240507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204419.304497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204419.304506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204419.304509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204419.304517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204419.304524:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204419.304530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204419.304535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204419.304538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204419.304541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204419.496538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204419.496546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204419.496553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204419.496555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204419.496559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204419.496560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204419.496563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204419.496564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204419.496651:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204419.496655:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204419.496658:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204420.264481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204420.264489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204420.264491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204420.264494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204420.264497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204420.264499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204420.264501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204420.264502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204420.264504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204420.328475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204420.328482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204420.328487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204420.328494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204420.328500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204420.328506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204420.328511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204420.328514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204420.328516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204420.520505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204420.520510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204420.520514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204420.520515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204420.520516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204420.520517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204420.520519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204420.520520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204420.520535:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204420.520537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204420.520540:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204420.840523:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204420.840542:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204420.840545:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204420.840562:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204420.840567:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204420.840571:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204420.840573:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204420.840577:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204420.840580:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204421.288434:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204421.288447:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204421.288450:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204421.288458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204421.288462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204421.288465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204421.288468:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204421.288471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204421.288474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204421.352437:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204421.352444:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204421.352446:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204421.352451:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204421.352454:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204421.352458:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204421.352466:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204421.352468:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204421.352469:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204421.544484:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204421.544487:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204421.544491:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544495:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204421.544498:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544499:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544501:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204421.544506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204421.544511:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204421.544513:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204421.544514:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544524:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204421.544525:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544527:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544529:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204421.544534:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204421.544535:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204421.544536:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544566:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204421.544568:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544569:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204421.544571:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204421.544625:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204421.544629:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204421.544631:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204421.864488:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300548610, at 4300549120 00000100:00000040:1.0:1788204421.864488:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300548610, at 4300549120 00000100:00000010:0.0:1788204421.864502:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000010:1.0:1788204421.864505:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000001:1.0:1788204422.312471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204422.312477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204422.312478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204422.312482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204422.312485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204422.312487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204422.312489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204422.312490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204422.312492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204422.376469:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204422.376480:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204422.376486:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204422.376489:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204422.376495:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204422.376503:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5877 00000400:00000200:1.0:1788204422.376512:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204422.376518:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204422.376522:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204422.376538:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376542:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.376545:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376548:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204422.376552:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204422.376555:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204422.376561:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204422.376567:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204422.376572:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204422.376579:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204422.376586:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376589:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.376591:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376594:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204422.376597:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204422.376601:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376604:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204422.376607:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204422.376610:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204422.376613:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.376616:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204422.376618:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204422.376627:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788204422.376631:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204422.376635:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204422.376637:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.376640:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204422.376642:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204422.376646:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204422.376654:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be800. 00000100:00000001:1.0:1788204422.376661:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204422.376671:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205860992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204422.376680:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204422.376685:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204422.376689:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205860992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204422.376712:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204422.376715:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204422.376722:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204422.376724:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204422.376725:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204422.376728:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204422.376731:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5877 00000400:00000200:1.0:1788204422.376735:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204422.376739:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204422.376741:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204422.376745:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376749:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.376765:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376771:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204422.376776:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204422.376780:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204422.376783:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204422.376787:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204422.376792:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204422.376799:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204422.376807:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376812:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.376817:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376822:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204422.376827:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204422.376833:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204422.376839:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204422.376844:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204422.376851:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204422.376857:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.376862:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204422.376869:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204422.376875:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.376877:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204422.376883:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000020:00000040:1.0:1788204422.376887:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204422.376901:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204422.376904:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.376907:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204422.376911:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204422.376915:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204422.376922:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788204422.376929:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204422.376939:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205861120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204422.376954:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204422.376959:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204422.376967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788204422.376968:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205861120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1788204422.376971:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1788204422.377001:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204422.377005:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204422.377010:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204422.377014:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204422.377015:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204422.377020:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204422.377026:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5877 00000400:00000200:1.0:1788204422.377033:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204422.377040:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204422.377045:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204422.377052:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377058:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.377062:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377065:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204422.377071:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204422.377075:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204422.377078:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204422.377081:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204422.377085:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204422.377089:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204422.377094:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377098:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.377104:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377109:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204422.377113:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204422.377120:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377125:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204422.377129:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204422.377134:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204422.377138:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377142:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204422.377146:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204422.377151:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377155:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204422.377170:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788204422.377177:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204422.377181:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204422.377185:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377189:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204422.377193:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204422.377197:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204422.377204:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788204422.377210:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204422.377219:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205861248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204422.377244:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204422.377258:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204422.377268:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205861248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204422.377285:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204422.377290:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204422.377295:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204422.377299:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204422.377301:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204422.377306:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204422.377311:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5877 00000400:00000200:1.0:1788204422.377320:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204422.377327:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204422.377331:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204422.377339:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377343:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.377350:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377355:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204422.377361:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204422.377389:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204422.377393:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204422.377398:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204422.377404:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204422.377409:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204422.377418:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377424:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204422.377430:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377437:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204422.377443:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204422.377450:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204422.377458:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204422.377462:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204422.377468:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204422.377472:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377476:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204422.377479:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204422.377484:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377488:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204422.377493:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204422.377499:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204422.377506:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204422.377509:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204422.377513:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204422.377516:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204422.377520:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204422.377539:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788204422.377546:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204422.377558:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205861376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204422.377580:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204422.377585:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204422.377595:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205861376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1788204422.377603:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204422.377605:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1788204422.377619:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204422.377627:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5887) 00000800:00000200:2.1:1788204422.377801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204422.377802:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204422.378040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204422.378042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204422.568495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204422.568500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204422.568504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204422.568505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204422.568507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204422.568508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204422.568510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204422.568510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204422.568551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204422.568554:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204422.568556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204423.336486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204423.336494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204423.336496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204423.336501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204423.336505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204423.336521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204423.336524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204423.336526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204423.336529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204423.592475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204423.592482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204423.592487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204423.592489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204423.592491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204423.592492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204423.592495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204423.592496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204423.592565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204423.592569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204423.592571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204424.360474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204424.360480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204424.360481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204424.360486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204424.360488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204424.360512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204424.360514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204424.360524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204424.360527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204424.424486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204424.424501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204424.424504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204424.424508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204424.424517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204424.424520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204424.424523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204424.424524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204424.424526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204424.616525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204424.616531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204424.616536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204424.616538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204424.616541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204424.616542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204424.616545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204424.616546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204424.616563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204424.616565:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204424.616567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204425.384491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204425.384498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204425.384501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204425.384506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204425.384509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204425.384513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204425.384516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204425.384518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204425.384521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204425.448463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204425.448496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204425.448500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204425.448505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204425.448509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204425.448515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204425.448526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204425.448530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204425.448533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204425.640497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204425.640506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204425.640513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204425.640514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204425.640516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204425.640517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204425.640520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204425.640521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204425.640580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204425.640584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204425.640589:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204426.408464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204426.408471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204426.408473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204426.408478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204426.408482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204426.408486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204426.408489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204426.408491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204426.408494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204426.472485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204426.472494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204426.472497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204426.472505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204426.472511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204426.472517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204426.472521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204426.472525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204426.472529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204426.664521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204426.664526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204426.664530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204426.664530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204426.664533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204426.664534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204426.664536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204426.664538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204426.664554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204426.664557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204426.664560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204426.984492:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300553729, at 4300554240 00000100:00000040:0.0:1788204426.984499:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300553729, at 4300554240 00000100:00000010:1.0:1788204426.984508:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000010:0.0:1788204426.984510:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000001:1.0:1788204427.432466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204427.432473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204427.432475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204427.432479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204427.432483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204427.432486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204427.432489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204427.432491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204427.432494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204427.496459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204427.496466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204427.496470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204427.496473:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204427.496477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204427.496483:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204427.496484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204427.496487:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204427.496490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204427.496492:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204427.496497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204427.496499:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204427.496501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204427.496507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204427.496507:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5882 00000400:00000200:1.0:1788204427.496520:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204427.496527:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204427.496531:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204427.496535:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496539:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.496542:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496544:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204427.496547:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204427.496549:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204427.496552:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204427.496554:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204427.496557:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204427.496559:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204427.496562:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496564:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.496566:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496568:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204427.496571:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204427.496573:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496576:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204427.496579:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204427.496582:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204427.496585:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.496587:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204427.496589:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204427.496598:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204427.496602:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204427.496606:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204427.496608:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.496611:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204427.496613:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204427.496616:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204427.496624:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204427.496630:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204427.496639:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205861504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204427.496649:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204427.496653:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204427.496657:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205861504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204427.496678:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204427.496681:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204427.496684:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204427.496685:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204427.496687:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204427.496690:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204427.496692:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5882 00000400:00000200:1.0:1788204427.496697:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204427.496700:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204427.496703:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204427.496706:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496710:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.496715:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496720:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204427.496724:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204427.496727:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204427.496730:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204427.496734:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204427.496739:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204427.496744:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204427.496753:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496758:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.496763:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496768:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204427.496772:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204427.496777:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496783:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204427.496787:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204427.496792:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204427.496796:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.496798:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204427.496807:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204427.496813:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.496816:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204427.496823:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788204427.496827:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204427.496832:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204427.496834:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.496839:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204427.496842:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204427.496844:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204427.496851:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:1.0:1788204427.496857:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204427.496865:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205861632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204427.496876:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204427.496880:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204427.496885:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205861632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204427.496899:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204427.496903:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204427.496909:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204427.496913:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204427.496916:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204427.496922:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204427.496927:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5882 00000400:00000200:1.0:1788204427.496942:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204427.496949:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204427.496953:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204427.496959:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496964:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1788204427.496968:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788204427.496970:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496973:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204427.496975:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204427.496976:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204427.496978:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204427.496981:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000200:2.1:1788204427.496983:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1788204427.496985:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204427.496990:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204427.496995:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.496997:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.496998:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497001:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204427.497005:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204427.497008:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497012:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204427.497016:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204427.497020:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204427.497023:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497026:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204427.497029:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204427.497034:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497044:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204427.497050:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000020:00000040:1.0:1788204427.497054:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204427.497059:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204427.497062:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497066:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204427.497068:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204427.497071:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204427.497076:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788204427.497081:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204427.497091:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205861760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204427.497101:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204427.497105:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204427.497110:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205861760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204427.497124:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204427.497129:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204427.497134:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204427.497137:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204427.497140:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204427.497145:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204427.497149:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5882 00000400:00000200:1.0:1788204427.497156:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204427.497162:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204427.497165:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204427.497170:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497173:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.497177:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497182:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204427.497186:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204427.497188:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204427.497190:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204427.497194:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204427.497197:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204427.497200:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204427.497207:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497210:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204427.497214:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497216:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204427.497220:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204427.497225:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204427.497230:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204427.497233:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204427.497237:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204427.497241:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497244:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204427.497248:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204427.497254:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497257:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204427.497263:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788204427.497267:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204427.497274:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204427.497278:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204427.497282:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204427.497286:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204427.497289:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204427.497295:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788204427.497300:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204427.497307:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205861888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204427.497331:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204427.497335:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204427.497344:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205861888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204427.497356:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204427.497387:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5892) 00000800:00000200:2.1:1788204427.497521:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204427.497523:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204427.497702:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204427.497705:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204427.497913:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204427.497915:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00010000:00000001:1.0:1788204427.688444:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204427.688451:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204427.688457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204427.688458:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204427.688461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204427.688462:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204427.688466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204427.688467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204427.688526:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204427.688529:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204427.688532:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204428.456478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204428.456485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204428.456487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204428.456492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204428.456496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204428.456499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204428.456502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204428.456504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204428.456507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204428.520489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204428.520496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204428.520500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204428.520512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204428.520518:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204428.520521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204428.520524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204428.520528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204428.520531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204428.712512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204428.712518:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204428.712520:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204428.712522:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712524:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204428.712525:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712526:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712527:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204428.712530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204428.712532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204428.712533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204428.712545:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204428.712547:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204428.712548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204429.480518:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204429.480526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204429.480528:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204429.480533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204429.480537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204429.480540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204429.480542:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204429.480545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204429.480548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204429.544494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204429.544499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204429.544502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204429.544509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204429.544514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204429.544519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204429.544532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204429.544534:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204429.544538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204429.736503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204429.736509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204429.736516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204429.736518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204429.736521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204429.736523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204429.736527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204429.736529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204429.736589:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204429.736594:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204429.736597:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204430.504446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204430.504453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204430.504455:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204430.504459:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204430.504461:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204430.504464:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204430.504465:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204430.504467:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204430.504469:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204430.568480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204430.568487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204430.568489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204430.568495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204430.568499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204430.568503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204430.568506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204430.568509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204430.568514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204430.760529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204430.760535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204430.760540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204430.760541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204430.760545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204430.760546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204430.760549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204430.760550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204430.760568:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204430.760571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204430.760574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204431.528464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204431.528471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204431.528473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204431.528478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204431.528481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204431.528485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204431.528487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204431.528490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204431.528493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204431.592477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204431.592485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204431.592489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204431.592498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204431.592506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204431.592512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204431.592518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204431.592523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204431.592527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204431.784504:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204431.784507:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204431.784510:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784514:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204431.784517:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784519:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784521:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204431.784527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204431.784531:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204431.784532:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204431.784533:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784535:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204431.784536:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784537:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784539:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204431.784543:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204431.784544:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204431.784546:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784547:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204431.784548:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784550:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204431.784552:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204431.784599:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204431.784602:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204431.784604:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204432.104453:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300558849, at 4300559360 00000100:00000040:1.0:1788204432.104453:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300558849, at 4300559360 00000100:00000010:1.0:1788204432.104467:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000010:0.0:1788204432.104469:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:1.0:1788204432.552460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204432.552466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204432.552468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204432.552472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204432.552474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204432.552477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204432.552479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204432.552481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204432.552483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204432.616474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788204432.616479:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204432.616484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204432.616486:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204432.616490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616490:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204432.616494:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204432.616498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204432.616500:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204432.616504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788204432.616509:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5887 00000100:00000001:0.0:1788204432.616519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204432.616523:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204432.616525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204432.616529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204432.616539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204432.616548:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204432.616557:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204432.616572:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616577:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.616580:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616582:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204432.616586:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204432.616588:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204432.616592:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204432.616596:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204432.616599:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204432.616601:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204432.616605:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616607:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.616609:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616610:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204432.616613:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204432.616617:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616619:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204432.616622:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204432.616625:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204432.616628:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616630:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204432.616632:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204432.616641:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788204432.616645:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204432.616650:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204432.616651:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616654:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204432.616656:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204432.616660:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204432.616667:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be400. 00000100:00000001:1.0:1788204432.616674:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204432.616683:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205862016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204432.616693:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204432.616697:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204432.616702:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205862016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204432.616720:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204432.616722:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204432.616725:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204432.616726:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204432.616727:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204432.616730:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204432.616731:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5887 00000400:00000200:1.0:1788204432.616734:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204432.616736:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204432.616738:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204432.616740:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616741:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.616743:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616744:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204432.616746:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204432.616747:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204432.616748:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204432.616750:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204432.616751:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204432.616753:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204432.616754:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616756:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.616758:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616762:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204432.616767:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204432.616773:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616779:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204432.616783:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204432.616790:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204432.616796:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616800:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204432.616807:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204432.616812:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616816:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204432.616823:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000020:00000040:1.0:1788204432.616828:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204432.616834:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204432.616838:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.616842:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204432.616844:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204432.616849:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204432.616857:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:1.0:1788204432.616864:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204432.616874:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205862144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204432.616887:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204432.616891:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204432.616899:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205862144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204432.616917:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204432.616922:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204432.616927:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204432.616929:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204432.616933:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204432.616939:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204432.616944:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5887 00000400:00000200:1.0:1788204432.616953:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204432.616957:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204432.616960:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204432.616965:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616970:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.616977:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.616982:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204432.616986:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204432.616991:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204432.616994:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204432.616998:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204432.617003:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204432.617008:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204432.617015:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617019:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.617025:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204432.617030:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204432.617031:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:2.1:1788204432.617034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000020:00000040:1.0:1788204432.617035:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204432.617041:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617045:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204432.617048:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204432.617051:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204432.617054:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617056:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204432.617059:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204432.617064:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617066:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204432.617069:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204432.617074:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204432.617080:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204432.617081:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617084:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204432.617087:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204432.617090:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204432.617098:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788204432.617105:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204432.617114:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205862272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204432.617136:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204432.617141:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204432.617147:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205862272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204432.617162:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204432.617167:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204432.617173:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204432.617177:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204432.617179:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204432.617184:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204432.617188:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5887 00000400:00000200:1.0:1788204432.617196:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204432.617201:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204432.617204:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204432.617210:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617213:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.617218:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617222:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204432.617225:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204432.617227:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204432.617229:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204432.617231:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204432.617235:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204432.617238:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204432.617244:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617248:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204432.617253:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617258:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204432.617262:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204432.617265:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204432.617267:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204432.617270:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204432.617274:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204432.617277:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617279:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204432.617282:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204432.617286:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617290:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204432.617295:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788204432.617299:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204432.617302:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204432.617304:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204432.617307:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204432.617308:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204432.617310:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204432.617315:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788204432.617319:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204432.617326:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205862400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204432.617338:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204432.617342:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204432.617361:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205862400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204432.617392:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204432.617396:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5897) 00000800:00000200:2.1:1788204432.617602:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204432.617604:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204432.617849:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204432.617851:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204432.618040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204432.618042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00010000:00000001:1.0:1788204432.808499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204432.808510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204432.808518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204432.808520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204432.808523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204432.808525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204432.808529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204432.808530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204432.808561:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204432.808564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204432.808567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204433.576488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204433.576497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204433.576499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204433.576504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204433.576508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204433.576512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204433.576515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204433.576517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204433.576521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204433.640472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204433.640479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204433.640483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204433.640492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204433.640498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204433.640504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204433.640509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204433.640513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204433.640515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204433.832522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204433.832532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204433.832540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204433.832542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204433.832546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204433.832547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204433.832551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204433.832553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204433.832621:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204433.832625:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204433.832629:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204434.600469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204434.600474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204434.600476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204434.600479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204434.600481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204434.600484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204434.600486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204434.600488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204434.600489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204434.664475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204434.664485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204434.664491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204434.664501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204434.664509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204434.664517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204434.664531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204434.664536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204434.664539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204434.856512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204434.856518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204434.856523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204434.856524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204434.856527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204434.856528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204434.856531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204434.856533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204434.856563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204434.856566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204434.856570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204435.624486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204435.624493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204435.624495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204435.624500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204435.624503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204435.624507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204435.624510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204435.624513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204435.624516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204435.688483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204435.688493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204435.688497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204435.688503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204435.688509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204435.688516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204435.688522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204435.688527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204435.688530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204435.880475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204435.880481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204435.880485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204435.880486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204435.880489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204435.880490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204435.880492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204435.880493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204435.880549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204435.880553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204435.880556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204436.648467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204436.648474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204436.648476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204436.648479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204436.648482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204436.648484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204436.648486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204436.648488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204436.648490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204436.712485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204436.712496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204436.712500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204436.712507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204436.712516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204436.712529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204436.712534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204436.712537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204436.712542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204436.904544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204436.904552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204436.904560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204436.904561:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204436.904565:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204436.904566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204436.904570:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204436.904571:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204436.904598:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204436.904601:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204436.904604:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204437.224490:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300563969, at 4300564480 00000100:00000040:1.0:1788204437.224490:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300563969, at 4300564480 00000100:00000010:1.0:1788204437.224506:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000010:0.0:1788204437.224508:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000001:1.0:1788204437.672476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204437.672490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204437.672492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204437.672497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204437.672501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204437.672504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204437.672507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204437.672509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204437.672512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204437.736468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204437.736476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204437.736479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204437.736486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204437.736491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204437.736496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204437.736499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204437.736503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204437.736508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204437.736537:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204437.736542:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204437.736545:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204437.736551:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204437.736556:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204437.736560:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204437.736565:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204437.736569:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204437.736575:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204437.737044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204437.737054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204437.737506:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204437.737508:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204437.737780:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204437.737783:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204437.738046:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204437.738048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00010000:00000001:1.0:1788204437.928477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204437.928484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204437.928490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204437.928491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204437.928494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204437.928496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204437.928500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204437.928501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204437.928567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204437.928571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204437.928573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204438.696464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204438.696470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204438.696471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204438.696475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204438.696477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204438.696480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204438.696482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204438.696483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204438.696485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204438.760534:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204438.760547:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204438.760548:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204438.760552:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204438.760556:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204438.760559:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204438.760561:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204438.760563:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204438.760565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204438.952552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204438.952564:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204438.952566:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204438.952576:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952579:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204438.952582:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952584:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952586:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952591:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204438.952593:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204438.952597:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204438.952599:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204438.952619:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204438.952622:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204438.952625:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204439.720508:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204439.720519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204439.720521:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204439.720529:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204439.720533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204439.720538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204439.720541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204439.720544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204439.720548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204439.784571:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204439.784579:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204439.784582:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204439.784587:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204439.784591:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204439.784594:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204439.784597:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204439.784600:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204439.784603:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204439.976543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204439.976554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204439.976562:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204439.976563:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204439.976567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204439.976568:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204439.976572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204439.976573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204439.976701:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204439.976706:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204439.976710:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204440.744436:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204440.744442:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204440.744443:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204440.744447:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204440.744450:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204440.744452:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204440.744454:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204440.744456:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204440.744458:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204440.808479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204440.808484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204440.808487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204440.808491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204440.808494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204440.808497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204440.808500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204440.808502:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204440.808505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204441.000529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204441.000535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204441.000540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204441.000541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204441.000544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204441.000546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204441.000549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204441.000551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204441.000569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204441.000572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204441.000575:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204441.768584:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204441.768595:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204441.768597:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204441.768604:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204441.768609:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204441.768614:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204441.768616:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204441.768619:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204441.768622:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204441.832547:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204441.832556:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204441.832559:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204441.832566:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204441.832574:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204441.832578:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204441.832581:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204441.832583:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204441.832586:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204442.024508:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204442.024511:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204442.024516:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024522:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204442.024525:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024527:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024530:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204442.024537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204442.024542:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204442.024543:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204442.024544:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024546:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204442.024547:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024549:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024551:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204442.024556:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204442.024557:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204442.024558:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024560:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204442.024561:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024563:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204442.024565:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204442.024626:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204442.024631:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204442.024635:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204442.344488:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300569089, at 4300569600 00000100:00000040:0.0:1788204442.344489:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300569089, at 4300569600 00000100:00000010:0.0:1788204442.344510:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000010:1.0:1788204442.344515:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000001:1.0:1788204442.792431:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204442.792436:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204442.792438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204442.792441:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204442.792443:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204442.792446:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204442.792448:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204442.792450:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204442.792451:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204442.856516:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204442.856528:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204442.856534:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204442.856542:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204442.856552:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204442.856564:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5897 00000400:00000200:1.0:1788204442.856585:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204442.856597:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204442.856602:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204442.856606:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856612:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.856616:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856618:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204442.856622:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204442.856624:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204442.856627:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204442.856630:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204442.856632:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204442.856634:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204442.856637:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856639:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.856640:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856642:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204442.856645:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204442.856648:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856649:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204442.856651:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204442.856655:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204442.856657:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.856658:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204442.856661:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204442.856670:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204442.856673:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204442.856676:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204442.856677:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.856680:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204442.856683:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204442.856686:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204442.856694:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:1.0:1788204442.856700:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204442.856708:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205863040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204442.856715:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204442.856719:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204442.856722:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205863040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204442.856744:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204442.856746:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204442.856749:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204442.856750:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204442.856752:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204442.856754:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204442.856756:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5897 00000400:00000200:1.0:1788204442.856760:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204442.856765:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204442.856767:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204442.856769:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856770:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.856773:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856775:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204442.856779:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204442.856780:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204442.856782:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204442.856790:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204442.856793:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204442.856794:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204442.856802:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856811:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.856822:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856830:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204442.856837:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204442.856850:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204442.856855:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204442.856860:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204442.856874:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204442.856881:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.856885:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204442.856893:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204442.856900:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.856905:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204442.856921:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788204442.856926:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204442.856930:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204442.856932:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.856934:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204442.856936:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204442.856939:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204442.856944:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204442.856948:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204442.856957:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205863168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204442.856975:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204442.856978:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204442.856985:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205863168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204442.857023:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204442.857029:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204442.857036:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204442.857039:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204442.857047:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204442.857059:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204442.857066:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5897 00000400:00000200:1.0:1788204442.857078:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204442.857084:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204442.857086:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204442.857093:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857096:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.857099:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857101:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204442.857105:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204442.857108:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204442.857111:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204442.857113:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204442.857117:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204442.857120:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204442.857124:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857131:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.857136:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857138:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204442.857143:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204442.857152:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857156:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204442.857163:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204442.857166:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204442.857186:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857188:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204442.857190:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204442.857192:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857193:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204442.857198:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788204442.857203:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204442.857210:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204442.857212:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857215:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204442.857220:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204442.857225:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204442.857232:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788204442.857240:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204442.857254:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205863296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204442.857277:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204442.857283:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204442.857293:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205863296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1788204442.857302:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204442.857309:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204442.857312:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204442.857315:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204442.857319:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204442.857323:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204442.857327:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204442.857330:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5897 00000400:00000200:1.0:1788204442.857336:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204442.857340:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204442.857344:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204442.857351:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857358:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.857399:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857403:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204442.857409:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204442.857411:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204442.857416:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204442.857420:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204442.857424:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204442.857430:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204442.857434:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857440:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204442.857445:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857452:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204442.857458:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204442.857467:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204442.857472:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204442.857477:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204442.857483:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204442.857487:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857490:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204442.857500:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204442.857510:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857513:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204442.857521:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204442.857527:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204442.857531:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204442.857534:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204442.857541:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204442.857543:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204442.857547:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204442.857554:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788204442.857559:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204442.857566:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205863424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204442.857583:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204442.857590:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204442.857603:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205863424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204442.857622:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204442.857634:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5907) 00000800:00000200:2.1:1788204442.857720:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204442.857842:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204442.857986:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00010000:00000001:1.0:1788204443.048548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204443.048556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204443.048563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204443.048564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204443.048567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204443.048569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204443.048572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204443.048573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204443.048592:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204443.048595:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204443.048599:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204443.816540:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204443.816552:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204443.816554:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204443.816561:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204443.816566:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204443.816578:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204443.816582:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204443.816587:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204443.816591:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204444.072527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204444.072537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204444.072545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204444.072547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204444.072551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204444.072552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204444.072556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204444.072557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204444.072690:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204444.072693:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204444.072695:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204444.840561:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204444.840569:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204444.840571:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204444.840576:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204444.840581:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204444.840586:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204444.840667:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204444.840670:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204444.840674:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204444.904531:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204444.904541:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204444.904544:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204444.904550:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204444.904554:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204444.904558:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204444.904561:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204444.904565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204444.904568:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204445.096510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204445.096519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204445.096526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204445.096527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204445.096531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204445.096532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204445.096536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204445.096538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204445.096559:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204445.096562:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204445.096565:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204445.864514:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204445.864522:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204445.864523:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204445.864528:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204445.864531:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204445.864535:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204445.864538:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204445.864540:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204445.864542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204445.928884:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:0.0:1788204445.928935:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204445.928947:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204445.928951:0:2335:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:0.0:1788204445.928957:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204445.928963:0:2335:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788204445.928969:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204445.928974:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4e00. 00000400:00000200:0.0:1788204445.928980:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204445.928985:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204445.928993:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85880 00000400:00000010:0.0:1788204445.928996:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85880. 00000100:00000001:0.0:1788204445.929002:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204445.929014:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310536a00 x1875062205863040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929047:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204445.929052:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205863040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929061:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204445.929063:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204445.929074:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204445.929076:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4800. 00000400:00000200:0.0:1788204445.929079:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204445.929083:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204445.929086:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85088 00000400:00000010:0.0:1788204445.929087:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85088. 00000100:00000001:0.0:1788204445.929090:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204445.929094:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310537100 x1875062205863168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929101:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204445.929104:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205863168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929109:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204445.929111:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204445.929132:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204445.929135:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4300. 00000400:00000200:0.0:1788204445.929139:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204445.929142:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204445.929145:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:0.0:1788204445.929146:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204445.929149:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204445.929154:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310536300 x1875062205863296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929161:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204445.929164:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205863296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929169:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204445.929171:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204445.929175:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204445.929177:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4600. 00000400:00000200:0.0:1788204445.929180:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204445.929182:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204445.929185:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85c38 00000400:00000010:0.0:1788204445.929187:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85c38. 00000100:00000001:0.0:1788204445.929189:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204445.929193:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310535880 x1875062205863424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929200:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204445.929203:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205863424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204445.929209:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204445.929210:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1788204446.120501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204446.120509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204446.120516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204446.120518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204446.120522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204446.120524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204446.120529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204446.120531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204446.120586:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204446.120588:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204446.120591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204446.888491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204446.888498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204446.888500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204446.888505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204446.888508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204446.888512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204446.888514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204446.888517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204446.888520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204447.144526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204447.144534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204447.144539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204447.144541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204447.144544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204447.144546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204447.144549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204447.144550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204447.144568:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204447.144571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204447.144573:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204447.464472:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300574210, at 4300574720 00000100:00000040:0.0:1788204447.464475:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300574209, at 4300574720 00000100:00000010:1.0:1788204447.464488:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000010:0.0:1788204447.464489:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000001:1.0:1788204447.912489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204447.912496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204447.912497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204447.912507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204447.912509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204447.912512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204447.912514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204447.912516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204447.912517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1F:1788204447.977564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204447.977583:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204447.977944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204447.977946:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204447.978216:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204447.978218:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204447.978498:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204447.978500:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00010000:00000001:1.0:1788204448.168475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204448.168481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204448.168490:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204448.168492:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204448.168496:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168499:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204448.168501:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168505:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204448.168510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204448.168513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204448.168515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204448.168563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204448.168567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204448.168570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204448.936527:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204448.936535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204448.936537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204448.936541:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204448.936545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204448.936548:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204448.936550:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204448.936552:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204448.936555:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204449.192532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204449.192539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204449.192543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204449.192544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204449.192547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204449.192548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204449.192551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204449.192551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204449.192575:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204449.192577:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204449.192581:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204449.960467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204449.960477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204449.960479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204449.960485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204449.960490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204449.960495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204449.960498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204449.960501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204449.960505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204450.216605:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204450.216618:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204450.216626:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204450.216627:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204450.216630:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204450.216631:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204450.216635:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204450.216636:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204450.216699:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204450.216705:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204450.216708:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204450.984465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204450.984470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204450.984472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204450.984476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204450.984478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204450.984481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204450.984483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204450.984484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204450.984486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204451.240528:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204451.240530:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204451.240534:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240539:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204451.240541:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240544:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240546:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204451.240552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204451.240556:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204451.240558:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204451.240559:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240561:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204451.240562:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240564:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240565:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204451.240570:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204451.240571:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204451.240572:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240574:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204451.240575:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240577:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204451.240579:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204451.240599:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204451.240603:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204451.240606:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204452.008454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204452.008459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204452.008460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204452.008464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204452.008466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204452.008469:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204452.008470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204452.008472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204452.008474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204452.264451:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204452.264457:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204452.264463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204452.264464:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204452.264468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204452.264470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204452.264473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204452.264474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204452.264525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204452.264529:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204452.264531:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204452.584478:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300579330, at 4300579840 00000100:00000010:0.0:1788204452.584493:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000040:1.0:1788204452.584582:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300579330, at 4300579840 00000100:00000010:1.0:1788204452.584588:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000001:1.0:1788204453.032544:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204453.032550:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204453.032552:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204453.032555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204453.032557:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204453.032567:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204453.032569:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204453.032570:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204453.032572:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204453.096559:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204453.096565:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204453.096569:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204453.096575:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204453.096580:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204453.096584:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204453.096587:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204453.096589:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204453.096592:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204453.097041:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204453.097048:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788204453.097502:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204453.097504:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788204453.097764:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204453.097767:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1788204453.098029:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204453.098031:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00010000:00000001:1.0:1788204453.288509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204453.288515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204453.288521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204453.288522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204453.288525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204453.288526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204453.288529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204453.288530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204453.288554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204453.288557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204453.288560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204454.056456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204454.056464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204454.056466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204454.056472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204454.056475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204454.056479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204454.056481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204454.056484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204454.056487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204454.120456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204454.120463:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204454.120465:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204454.120471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204454.120477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204454.120482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204454.120492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204454.120495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204454.120497:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204454.312468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204454.312475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204454.312481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204454.312483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204454.312486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204454.312487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204454.312490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204454.312491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204454.312544:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204454.312548:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204454.312550:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204455.080460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204455.080467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204455.080469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204455.080474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204455.080477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204455.080481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204455.080483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204455.080485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204455.080488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204455.144493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204455.144495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204455.144500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204455.144501:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204455.144503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204455.144504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204455.144509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204455.144510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204455.144514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204455.144515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204455.144520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204455.144520:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204455.144525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204455.144528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204455.144532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204455.144537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204455.144540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204455.144542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204455.336513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204455.336519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204455.336525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204455.336526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204455.336529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204455.336530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204455.336533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204455.336534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204455.336560:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204455.336563:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204455.336566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204456.104472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204456.104480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204456.104482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204456.104487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204456.104491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204456.104494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204456.104496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204456.104497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204456.104500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204456.168489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204456.168496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204456.168498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204456.168504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204456.168507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204456.168511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204456.168514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204456.168516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204456.168519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204456.168615:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204456.168618:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204456.168619:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204456.168622:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204456.168624:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204456.168626:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204456.168628:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204456.168629:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204456.168631:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204456.360476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204456.360480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204456.360484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204456.360485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204456.360487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204456.360488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204456.360492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204456.360493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204456.360553:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204456.360557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204456.360560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204457.128511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204457.128524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204457.128525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204457.128531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204457.128534:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204457.128538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204457.128540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204457.128543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204457.128545:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204457.192490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204457.192493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204457.192496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204457.192502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204457.192504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204457.192508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204457.192512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204457.192515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204457.192517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204457.192525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204457.192525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204457.192531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204457.192531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204457.192535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204457.192539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204457.192543:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204457.192546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204457.192548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204457.384556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204457.384563:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204457.384568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204457.384569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204457.384572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204457.384573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204457.384576:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204457.384577:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204457.384596:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204457.384599:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204457.384601:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204457.704467:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300584449, at 4300584960 00000100:00000040:1.0:1788204457.704477:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300584449, at 4300584960 00000100:00000010:0.0:1788204457.704482:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000010:1.0:1788204457.704500:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000001:1.0:1788204458.152476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204458.152494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204458.152496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204458.152503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204458.152508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204458.152513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204458.152517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204458.152520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204458.152523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204458.216499:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204458.216505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204458.216507:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204458.216510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204458.216516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788204458.216518:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204458.216520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204458.216524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204458.216529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204458.216530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204458.216535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204458.216540:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204458.216540:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204458.216553:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204458.216554:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204458.216567:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204458.216574:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204458.216581:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204458.217251:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204458.217257:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204458.218097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204458.218099:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204458.218404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204458.218407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204458.218676:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204458.218679:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00010000:00000001:1.0:1788204458.408482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204458.408492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204458.408505:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204458.408508:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204458.408513:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408517:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204458.408519:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408522:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408524:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204458.408529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204458.408533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204458.408534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204458.408620:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204458.408626:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204458.408632:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204459.176494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204459.176504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204459.176514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204459.176522:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204459.176527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204459.176532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204459.176535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204459.176538:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204459.176542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204459.240491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204459.240491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204459.240499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204459.240500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204459.240504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204459.240504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204459.240511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204459.240515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204459.240516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204459.240520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204459.240521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204459.240522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204459.240526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204459.240531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204459.240535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204459.240536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204459.240538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204459.240542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204459.432516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204459.432525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204459.432533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204459.432535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204459.432539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204459.432540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204459.432544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204459.432545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204459.432564:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204459.432566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204459.432569:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204460.200429:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204460.200437:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204460.200439:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204460.200444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204460.200447:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204460.200450:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204460.200452:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204460.200455:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204460.200457:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204460.264465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204460.264465:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204460.264473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204460.264473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204460.264476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204460.264476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204460.264482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204460.264483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204460.264487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204460.264487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204460.264491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204460.264494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204460.264498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204460.264498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204460.264501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204460.264502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204460.264502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204460.264504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204460.456485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204460.456492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204460.456498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204460.456499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204460.456501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204460.456502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204460.456505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204460.456506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204460.456569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204460.456573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204460.456577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204461.224478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204461.224484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204461.224485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204461.224489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204461.224492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204461.224494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204461.224496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204461.224498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204461.224501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204461.288525:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204461.288525:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204461.288538:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204461.288539:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204461.288544:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204461.288545:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204461.288552:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204461.288554:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204461.288560:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204461.288561:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204461.288567:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204461.288568:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204461.288575:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204461.288580:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204461.288584:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204461.288585:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204461.288596:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204461.288600:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204461.480524:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204461.480526:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204461.480528:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480531:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204461.480533:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480535:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480537:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204461.480540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204461.480543:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204461.480544:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204461.480545:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480546:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204461.480547:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480548:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480549:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204461.480552:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204461.480553:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204461.480554:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480555:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204461.480555:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480556:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204461.480557:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204461.480569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204461.480571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204461.480573:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204462.248499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204462.248506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204462.248508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204462.248512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204462.248515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204462.248518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204462.248520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204462.248522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204462.248525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204462.312465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204462.312465:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204462.312472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204462.312473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204462.312475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204462.312476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204462.312480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204462.312481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204462.312485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204462.312486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204462.312490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204462.312490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204462.312495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204462.312497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204462.312499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204462.312502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204462.312504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204462.312506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204462.504455:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204462.504461:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204462.504468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204462.504469:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204462.504472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204462.504474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204462.504477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204462.504478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204462.504526:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204462.504530:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204462.504532:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204462.824472:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300589570, at 4300590080 00000100:00000010:1.0:1788204462.824485:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000040:0.0:1788204462.824560:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300589570, at 4300590080 00000100:00000010:0.0:1788204462.824565:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000001:1.0:1788204463.272461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204463.272468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204463.272471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204463.272476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204463.272479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204463.272483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204463.272486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204463.272488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204463.272490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204463.336467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204463.336467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204463.336473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204463.336473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204463.336477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204463.336477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204463.336482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204463.336482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204463.336488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204463.336488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204463.336494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204463.336494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204463.336497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204463.336499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204463.336505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204463.336505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204463.336510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204463.336514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204463.336926:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204463.336931:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204463.337421:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204463.337422:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204463.337653:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204463.337655:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204463.337995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204463.337997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00010000:00000001:1.0:1788204463.528513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204463.528519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204463.528522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204463.528523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204463.528525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204463.528526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204463.528528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204463.528529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204463.528541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204463.528543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204463.528544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204463.848500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204463.848507:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204463.848508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204463.848513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204463.848516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204463.848519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204463.848521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204463.848523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204463.848524:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204464.296490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204464.296496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204464.296498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204464.296502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204464.296505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204464.296508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204464.296509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204464.296511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204464.296513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.360461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204464.360461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204464.360468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204464.360469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204464.360471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204464.360471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204464.360477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204464.360478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204464.360482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204464.360482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204464.360485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204464.360486:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204464.360489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204464.360491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204464.360493:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204464.360494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204464.360497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204464.360499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204464.552457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204464.552462:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204464.552466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204464.552468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204464.552470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204464.552471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204464.552474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204464.552475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204464.552569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204464.552572:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204464.552575:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204464.872478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204464.872485:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204464.872487:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.872492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204464.872495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204464.872499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204464.872501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204464.872504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204464.872506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204465.320460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204465.320466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204465.320467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204465.320471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204465.320474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204465.320477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204465.320479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204465.320480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204465.320482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204465.384459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204465.384460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204465.384466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204465.384466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204465.384471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204465.384471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204465.384477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204465.384477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204465.384481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204465.384482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204465.384487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204465.384487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204465.384492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204465.384495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204465.384497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204465.384498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204465.384500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204465.384502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204465.576506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204465.576511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204465.576514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204465.576515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204465.576517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204465.576518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204465.576520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204465.576520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204465.576534:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204465.576536:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204465.576537:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204465.896488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204465.896494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204465.896496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204465.896500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204465.896502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204465.896505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204465.896507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204465.896510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204465.896511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204466.344486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204466.344493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204466.344495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204466.344500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204466.344503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204466.344507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204466.344509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204466.344511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204466.344514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204466.408480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204466.408480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204466.408487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204466.408488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204466.408489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204466.408491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204466.408498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204466.408498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204466.408505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204466.408505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204466.408509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204466.408510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204466.408515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204466.408517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204466.408520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204466.408523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204466.408526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204466.408529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204466.600451:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204466.600456:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204466.600460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204466.600461:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204466.600464:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204466.600465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204466.600467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204466.600468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204466.600513:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204466.600517:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204466.600520:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204466.920484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204466.920490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204466.920492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204466.920497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204466.920500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204466.920504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204466.920506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204466.920508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204466.920511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204467.368460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204467.368466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204467.368469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204467.368474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204467.368477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204467.368480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204467.368482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204467.368484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204467.368487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204467.432474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204467.432474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204467.432481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204467.432482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204467.432485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204467.432485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204467.432493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204467.432494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204467.432500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204467.432504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204467.432506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204467.432508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204467.432510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204467.432511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204467.432515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204467.432516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204467.432518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204467.432521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204467.624526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204467.624532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204467.624538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204467.624539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204467.624542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204467.624543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204467.624547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204467.624548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204467.624564:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204467.624567:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204467.624570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204467.944480:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300594689, at 4300595200 00000100:00000010:1.0:1788204467.944498:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000040:0.0:1788204467.944510:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300594689, at 4300595200 00000100:00000010:0.0:1788204467.944520:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000001:1.0:1788204468.392459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204468.392465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204468.392467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204468.392471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204468.392474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204468.392476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204468.392478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204468.392480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204468.392481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204468.456469:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204468.456470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204468.456477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204468.456481:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204468.456482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456485:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204468.456490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204468.456491:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204468.456496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204468.456498:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204468.456508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204468.456509:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5923 00000100:00000001:0.0:1788204468.456520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1788204468.456520:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204468.456524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204468.456528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204468.456529:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204468.456533:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204468.456537:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456541:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.456545:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456547:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204468.456551:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204468.456553:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204468.456556:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204468.456559:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204468.456562:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204468.456564:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204468.456567:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456569:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.456570:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456572:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204468.456575:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204468.456578:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456581:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204468.456584:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204468.456586:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204468.456589:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456591:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204468.456594:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204468.456604:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204468.456608:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204468.456612:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204468.456614:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456617:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204468.456619:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204468.456624:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204468.456632:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be800. 00000100:00000001:1.0:1788204468.456639:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204468.456648:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205865600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204468.456658:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204468.456662:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204468.456667:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205865600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204468.456688:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204468.456690:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204468.456693:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204468.456695:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204468.456696:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204468.456699:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204468.456701:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5923 00000400:00000200:1.0:1788204468.456705:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204468.456708:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204468.456711:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204468.456714:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456717:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.456720:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456723:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204468.456735:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204468.456736:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204468.456738:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204468.456740:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204468.456742:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204468.456744:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204468.456747:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456749:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.456751:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456756:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204468.456760:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204468.456768:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456772:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204468.456776:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204468.456782:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204468.456788:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456793:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204468.456799:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204468.456804:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456808:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204468.456816:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788204468.456820:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204468.456827:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204468.456830:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.456834:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204468.456837:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204468.456839:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204468.456846:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788204468.456851:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204468.456860:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205865728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204468.456880:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204468.456885:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204468.456894:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205865728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204468.456912:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204468.456917:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204468.456922:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204468.456927:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204468.456930:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204468.456936:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204468.456942:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5923 00000400:00000200:1.0:1788204468.456951:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204468.456957:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204468.456963:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204468.456970:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456980:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.456987:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.456991:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204468.456997:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204468.457000:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204468.457004:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204468.457006:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204468.457012:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204468.457018:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.1:1788204468.457019:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788204468.457025:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457027:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.457029:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457032:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.1:1788204468.457034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000020:00000040:1.0:1788204468.457037:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204468.457045:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457050:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204468.457056:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204468.457070:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204468.457077:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457082:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204468.457086:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204468.457091:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457095:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204468.457102:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788204468.457108:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204468.457113:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204468.457115:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457118:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204468.457120:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204468.457123:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204468.457130:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000100:00000001:1.0:1788204468.457136:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204468.457145:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205865856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204468.457167:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204468.457172:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204468.457181:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205865856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204468.457194:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204468.457199:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204468.457206:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204468.457211:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204468.457215:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204468.457222:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204468.457227:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5923 00000400:00000200:1.0:1788204468.457236:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204468.457243:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204468.457249:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204468.457257:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457263:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.457269:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457274:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204468.457288:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204468.457292:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204468.457295:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204468.457299:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204468.457305:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204468.457310:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204468.457319:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457324:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204468.457331:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457334:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204468.457338:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204468.457345:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204468.457351:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204468.457355:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204468.457359:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204468.457383:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457386:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204468.457388:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204468.457392:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457397:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204468.457403:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204468.457409:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204468.457416:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204468.457419:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204468.457424:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204468.457428:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204468.457432:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204468.457440:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788204468.457445:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204468.457455:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205865984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204468.457469:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204468.457473:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204468.457483:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205865984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204468.457502:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204468.457509:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5933) 00000800:00000200:3.1:1788204468.457577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204468.457579:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204468.457835:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204468.457837:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204468.458150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204468.458152:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00010000:00000001:1.0:1788204468.648468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204468.648474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204468.648483:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204468.648486:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204468.648489:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648492:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204468.648494:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648495:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648497:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204468.648500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204468.648502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204468.648503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204468.648561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204468.648565:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204468.648568:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204469.416474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204469.416479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204469.416480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204469.416484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204469.416486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204469.416489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204469.416491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204469.416493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204469.416495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204469.480463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204469.480470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204469.480473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204469.480478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204469.480483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204469.480487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204469.480494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204469.480497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204469.480500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204469.672512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204469.672519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204469.672524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204469.672525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204469.672528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204469.672529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204469.672532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204469.672533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204469.672551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204469.672554:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204469.672556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204470.440479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204470.440486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204470.440489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204470.440494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204470.440498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204470.440502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204470.440505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204470.440507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204470.440510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204470.504418:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204470.504424:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204470.504426:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204470.504430:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204470.504432:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204470.504436:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204470.504439:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204470.504444:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204470.504449:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204470.696470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204470.696477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204470.696482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204470.696483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204470.696486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204470.696487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204470.696490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204470.696492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204470.696542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204470.696545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204470.696548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204471.464484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204471.464492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204471.464494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204471.464499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204471.464502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204471.464506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204471.464508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204471.464511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204471.464514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204471.528479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204471.528485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204471.528489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204471.528496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204471.528502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204471.528506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204471.528519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204471.528522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204471.528525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204471.720492:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204471.720494:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204471.720497:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720500:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204471.720502:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720504:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204471.720507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204471.720510:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204471.720511:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204471.720512:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720513:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204471.720514:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720515:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720516:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204471.720518:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204471.720519:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204471.720520:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720521:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204471.720521:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720522:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204471.720523:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204471.720536:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204471.720537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204471.720539:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204472.488471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204472.488478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204472.488480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204472.488484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204472.488488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204472.488492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204472.488495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204472.488497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204472.488501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204472.552419:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204472.552425:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204472.552427:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204472.552431:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204472.552434:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204472.552437:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204472.552438:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204472.552440:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204472.552444:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204472.744457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204472.744462:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204472.744466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204472.744468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204472.744469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204472.744470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204472.744474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204472.744475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204472.744534:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204472.744538:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204472.744541:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204473.064458:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300599809, at 4300600320 00000100:00000040:1.0:1788204473.064458:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300599809, at 4300600320 00000100:00000010:0.0:1788204473.064472:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000010:1.0:1788204473.064478:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000001:1.0:1788204473.512472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204473.512478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204473.512481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204473.512486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204473.512490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204473.512493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204473.512495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204473.512497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204473.512500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204473.576464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204473.576470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204473.576472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204473.576477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204473.576489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204473.576492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204473.576496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204473.576499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204473.576502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204473.577027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204473.577033:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204473.577625:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204473.577628:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204473.577906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204473.577909:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204473.578185:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204473.578187:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00010000:00000001:1.0:1788204473.768479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204473.768486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204473.768492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204473.768493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204473.768495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204473.768497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204473.768499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204473.768501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204473.768516:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204473.768519:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204473.768521:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204474.536463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204474.536469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204474.536470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204474.536474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204474.536476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204474.536479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204474.536481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204474.536483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204474.536485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204474.600479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204474.600486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204474.600491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204474.600499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204474.600505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204474.600512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204474.600526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204474.600529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204474.600532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204474.792469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204474.792476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204474.792481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204474.792482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204474.792486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204474.792488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204474.792492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204474.792493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204474.792549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204474.792553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204474.792557:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204475.560471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204475.560477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204475.560479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204475.560482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204475.560484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204475.560487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204475.560489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204475.560491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204475.560492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204475.624477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204475.624485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204475.624488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204475.624494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204475.624499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204475.624503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204475.624507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204475.624510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204475.624513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204475.816502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204475.816509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204475.816514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204475.816516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204475.816519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204475.816520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204475.816523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204475.816524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204475.816547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204475.816550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204475.816553:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204476.584484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204476.584491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204476.584493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204476.584498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204476.584502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204476.584506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204476.584508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204476.584511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204476.584513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204476.648481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204476.648488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204476.648490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204476.648495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204476.648499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204476.648502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204476.648505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204476.648507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204476.648510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204476.840465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204476.840472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204476.840479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204476.840481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204476.840485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204476.840486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204476.840489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204476.840491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204476.840558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204476.840562:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204476.840565:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204477.608457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204477.608463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204477.608465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204477.608471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204477.608475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204477.608479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204477.608481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204477.608484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204477.608487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204477.672472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204477.672480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204477.672482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204477.672491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204477.672498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204477.672505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204477.672519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204477.672522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204477.672525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204477.864502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204477.864508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204477.864512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204477.864513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204477.864515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204477.864516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204477.864518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204477.864518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204477.864530:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204477.864532:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204477.864534:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204478.184495:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300604929, at 4300605440 00000100:00000040:1.0:1788204478.184495:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300604929, at 4300605440 00000100:00000010:1.0:1788204478.184514:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000010:0.0:1788204478.184521:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000001:1.0:1788204478.632492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204478.632501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204478.632503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204478.632509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204478.632513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204478.632517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204478.632520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204478.632523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204478.632533:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204478.696516:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204478.696527:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204478.696529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204478.696531:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204478.696536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204478.696545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788204478.696545:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204478.696561:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204478.696562:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204478.696567:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204478.696570:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204478.696577:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204478.696578:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204478.696583:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204478.696591:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5933 00000400:00000200:1.0:1788204478.696602:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204478.696611:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204478.696615:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204478.696618:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696620:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696622:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696624:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204478.696626:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204478.696628:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204478.696630:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204478.696633:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204478.696636:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204478.696638:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204478.696642:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696643:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696646:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696648:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204478.696651:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204478.696655:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696657:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204478.696660:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204478.696664:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204478.696667:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696669:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204478.696671:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204478.696681:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788204478.696685:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204478.696688:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204478.696690:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696692:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204478.696695:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204478.696699:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204478.696707:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788204478.696713:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204478.696723:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205866624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204478.696731:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204478.696735:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204478.696740:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205866624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204478.696761:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204478.696764:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204478.696768:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204478.696769:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204478.696770:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204478.696772:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204478.696774:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5933 00000400:00000200:1.0:1788204478.696777:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204478.696779:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204478.696781:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204478.696782:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696785:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696786:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696788:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204478.696789:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204478.696790:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204478.696791:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204478.696793:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204478.696794:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204478.696796:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204478.696797:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696799:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696800:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696802:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204478.696804:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204478.696808:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696810:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204478.696813:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204478.696818:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204478.696821:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696823:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204478.696826:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204478.696831:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696835:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204478.696839:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204478.696843:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204478.696847:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204478.696849:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696852:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204478.696854:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204478.696856:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204478.696859:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788204478.696862:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204478.696866:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205866752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204478.696876:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204478.696878:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204478.696891:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205866752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204478.696900:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204478.696903:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204478.696906:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204478.696907:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204478.696909:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204478.696911:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204478.696915:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5933 00000400:00000200:1.0:1788204478.696919:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204478.696922:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204478.696926:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204478.696930:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696933:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696934:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696935:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204478.696939:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204478.696940:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204478.696941:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204478.696942:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204478.696944:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204478.696947:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204478.696952:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696954:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.696958:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696960:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204478.696965:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204478.696969:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204478.696975:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204478.696977:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204478.696980:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204478.696984:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696985:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204478.696989:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204478.696991:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.696994:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204478.696998:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000020:00000040:1.0:1788204478.697001:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204478.697002:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204478.697003:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.697007:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204478.697009:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204478.697011:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204478.697014:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000100:00000001:1.0:1788204478.697016:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204478.697020:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205866880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204478.697030:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204478.697033:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204478.697038:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205866880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204478.697042:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204478.697043:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204478.697045:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204478.697049:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204478.697051:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204478.697054:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204478.697058:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5933 00000400:00000200:1.0:1788204478.697064:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204478.697070:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204478.697073:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204478.697077:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.697081:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.697084:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.697088:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204478.697093:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204478.697094:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204478.697096:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204478.697098:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204478.697104:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204478.697109:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204478.697116:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.697122:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204478.697127:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204478.697131:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204478.697136:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204478.697144:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204478.697149:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204478.697153:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204478.697157:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1788204478.697158:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:1.0:1788204478.697163:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204478.697163:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1788204478.697169:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204478.697173:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204478.697180:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.697182:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204478.697188:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788204478.697193:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204478.697200:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204478.697202:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204478.697204:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204478.697206:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204478.697210:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204478.697215:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788204478.697220:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204478.697227:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205867008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204478.697241:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204478.697243:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204478.697247:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205867008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204478.697258:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204478.697270:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5943) 00000800:00000200:2.1:1788204478.697609:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204478.697612:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204478.697869:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204478.697871:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1788204478.698071:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204478.698073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00010000:00000001:1.0:1788204478.888474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204478.888481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204478.888491:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204478.888494:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204478.888497:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888500:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204478.888502:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888505:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204478.888508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204478.888511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204478.888512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204478.888600:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204478.888608:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204478.888613:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204479.656443:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204479.656453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204479.656455:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204479.656462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204479.656466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204479.656472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204479.656475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204479.656478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204479.656482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204479.720498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204479.720509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204479.720513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204479.720523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204479.720541:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204479.720546:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204479.720550:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204479.720553:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204479.720557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204479.912512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204479.912521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204479.912556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204479.912557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204479.912560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204479.912562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204479.912565:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204479.912566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204479.912587:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204479.912590:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204479.912594:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204480.680504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204480.680524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204480.680527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204480.680533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204480.680538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204480.680542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204480.680544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204480.680547:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204480.680549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204480.744486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204480.744494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204480.744497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204480.744505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204480.744521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204480.744527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204480.744530:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204480.744533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204480.744537:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204480.936498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204480.936518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204480.936526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204480.936527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204480.936529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204480.936530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204480.936533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204480.936534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204480.936592:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204480.936597:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204480.936600:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204481.704444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204481.704454:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204481.704457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204481.704464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204481.704468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204481.704473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204481.704477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204481.704481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204481.704485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204481.768424:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204481.768434:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204481.768437:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204481.768445:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204481.768451:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204481.768457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204481.768468:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204481.768471:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204481.768473:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204481.960547:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204481.960549:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204481.960553:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960559:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204481.960561:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960563:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960565:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960569:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204481.960570:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960574:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204481.960576:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204481.960577:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204481.960578:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960580:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204481.960581:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960582:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960584:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960588:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204481.960590:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204481.960591:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204481.960592:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960593:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204481.960594:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960596:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204481.960598:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204481.960619:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204481.960623:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204481.960626:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204482.728475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204482.728510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204482.728513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204482.728519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204482.728524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204482.728529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204482.728533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204482.728536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204482.728539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204482.792461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204482.792470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204482.792471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204482.792486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204482.792492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204482.792496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204482.792500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204482.792505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204482.792509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204482.984492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204482.984499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204482.984504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204482.984505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204482.984508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204482.984509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204482.984511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204482.984512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204482.984564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204482.984569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204482.984574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204483.304510:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300610049, at 4300610560 00000100:00000040:1.0:1788204483.304510:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300610049, at 4300610560 00000100:00000010:0.0:1788204483.304529:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000010:1.0:1788204483.304531:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000001:1.0:1788204483.752515:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204483.752526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204483.752528:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204483.752535:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204483.752539:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204483.752544:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204483.752547:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204483.752550:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204483.752554:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204483.816483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204483.816493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204483.816498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204483.816505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204483.816511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204483.816515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204483.816528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204483.816532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204483.816535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204483.817076:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204483.817091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204483.817655:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204483.817657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204483.817915:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204483.817918:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1788204483.818191:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204483.818194:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204484.008603:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204484.008613:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204484.008619:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204484.008620:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204484.008633:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204484.008634:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204484.008638:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204484.008640:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204484.008665:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204484.008668:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204484.008671:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204484.776470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204484.776479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204484.776481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204484.776486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204484.776489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204484.776493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204484.776494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204484.776496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204484.776499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204484.840459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204484.840471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204484.840537:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204484.840540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204484.840542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204484.840543:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204484.840545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204484.840546:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204484.840548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204485.032495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204485.032505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204485.032512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204485.032513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204485.032515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204485.032516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204485.032519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204485.032520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204485.032577:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204485.032582:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204485.032586:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204485.800513:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204485.800521:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204485.800523:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204485.800528:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204485.800533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204485.800538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204485.800541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204485.800545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204485.800549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204485.864484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204485.864492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204485.864497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204485.864504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204485.864510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204485.864514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204485.864527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204485.864530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204485.864534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204486.056534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204486.056543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204486.056551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204486.056552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204486.056555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204486.056556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204486.056558:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204486.056559:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204486.056578:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204486.056580:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204486.056582:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204486.824469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204486.824484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204486.824487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204486.824494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204486.824499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204486.824504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204486.824507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204486.824516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204486.824520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204486.888508:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204486.888517:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204486.888520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204486.888527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204486.888531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204486.888536:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204486.888545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204486.888547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204486.888549:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204487.080505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204487.080513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204487.080520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204487.080522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204487.080526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204487.080527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204487.080531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204487.080532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204487.080584:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204487.080588:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204487.080592:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204487.848535:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204487.848546:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204487.848548:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204487.848555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204487.848560:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204487.848564:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204487.848567:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204487.848570:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204487.848573:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204487.912456:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204487.912464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204487.912468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204487.912476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204487.912482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204487.912488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204487.912498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204487.912500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204487.912503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204488.104524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204488.104532:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204488.104535:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204488.104538:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104540:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204488.104542:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104544:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104545:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204488.104550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204488.104553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204488.104554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204488.104569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204488.104572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204488.104574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204488.424532:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300615169, at 4300615680 00000100:00000040:1.0:1788204488.424532:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300615169, at 4300615680 00000100:00000010:0.0:1788204488.424551:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000010:1.0:1788204488.424555:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000001:1.0:1788204488.872483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204488.872490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204488.872493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204488.872498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204488.872501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204488.872505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204488.872513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204488.872516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204488.872519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204488.936505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204488.936514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204488.936521:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204488.936534:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204488.936544:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204488.936551:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204488.936559:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204488.936565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204488.936570:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204488.936983:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204488.936987:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204488.937543:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204488.937545:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204488.937832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204488.937835:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204488.938099:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204488.938101:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204489.128472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204489.128477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204489.128480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204489.128481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204489.128483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204489.128484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204489.128486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204489.128487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204489.128552:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204489.128556:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204489.128559:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204489.896480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204489.896488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204489.896491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204489.896497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204489.896500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204489.896504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204489.896506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204489.896509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204489.896512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204489.960428:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204489.960435:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204489.960438:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204489.960444:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204489.960449:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204489.960454:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204489.960460:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204489.960464:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204489.960467:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204490.152501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204490.152508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204490.152513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204490.152514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204490.152517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204490.152518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204490.152522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204490.152523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204490.152541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204490.152544:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204490.152547:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204490.920476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204490.920483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204490.920485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204490.920492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204490.920495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204490.920500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204490.920502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204490.920503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204490.920505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204490.984468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204490.984476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204490.984480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204490.984486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204490.984492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204490.984499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204490.984512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204490.984514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204490.984517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176438:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204491.176445:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204491.176448:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204491.176451:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176456:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204491.176459:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176461:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176463:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204491.176475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204491.176478:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204491.176479:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204491.176480:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176481:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204491.176482:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176483:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176484:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204491.176487:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204491.176487:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204491.176488:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176489:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204491.176490:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176491:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204491.176492:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204491.176541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204491.176544:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204491.176547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204491.944466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204491.944473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204491.944475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204491.944481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204491.944484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204491.944489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204491.944491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204491.944503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204491.944506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204492.008423:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204492.008429:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204492.008432:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204492.008437:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204492.008441:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204492.008445:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204492.008449:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204492.008452:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204492.008456:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204492.200529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204492.200534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204492.200538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204492.200539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204492.200542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204492.200542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204492.200545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204492.200546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204492.200558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204492.200560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204492.200562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204492.968488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204492.968495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204492.968497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204492.968502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204492.968505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204492.968509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204492.968511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204492.968513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204492.968516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204493.032461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204493.032469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204493.032472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204493.032478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204493.032484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204493.032489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204493.032496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204493.032499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204493.032502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204493.224467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204493.224474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204493.224480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204493.224481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204493.224485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204493.224486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204493.224488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204493.224490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204493.224554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204493.224558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204493.224561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204493.544415:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300620289, at 4300620800 00000100:00000040:1.0:1788204493.544421:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300620289, at 4300620800 00000100:00000010:0.0:1788204493.544427:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000010:1.0:1788204493.544430:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000001:1.0:1788204493.992481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204493.992489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204493.992491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204493.992497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204493.992500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204493.992503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204493.992506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204493.992508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204493.992511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204494.056440:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204494.056446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204494.056450:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204494.056457:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204494.056463:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204494.056467:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204494.056472:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204494.056474:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204494.056479:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204494.056481:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204494.056483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056485:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204494.056490:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204494.056494:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204494.056501:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5948 00000400:00000200:0.0:1788204494.056510:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204494.056515:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204494.056519:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204494.056522:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056526:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.056529:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056531:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204494.056535:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204494.056537:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204494.056539:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204494.056542:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204494.056545:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204494.056547:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204494.056550:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056552:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.056554:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056556:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204494.056559:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204494.056562:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056565:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204494.056567:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204494.056570:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204494.056573:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056576:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204494.056578:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204494.056586:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204494.056589:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204494.056593:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204494.056595:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056598:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204494.056600:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204494.056604:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204494.056612:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204494.056618:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204494.056627:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205868160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204494.056637:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204494.056641:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204494.056646:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205868160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204494.056668:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204494.056671:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204494.056674:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204494.056676:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204494.056677:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204494.056680:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204494.056683:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5948 00000400:00000200:0.0:1788204494.056687:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204494.056692:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204494.056694:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204494.056698:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056703:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.056708:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056712:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204494.056716:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204494.056720:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204494.056723:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204494.056727:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204494.056732:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204494.056737:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204494.056744:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056749:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.056754:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056757:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204494.056762:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204494.056766:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056769:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204494.056772:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204494.056780:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204494.056785:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056789:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204494.056795:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204494.056800:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056806:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204494.056814:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204494.056819:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204494.056824:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204494.056827:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.056829:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204494.056831:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204494.056835:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204494.056841:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204494.056846:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204494.056857:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205868288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204494.056882:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204494.056888:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204494.056897:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205868288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204494.056914:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204494.056918:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204494.056925:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204494.056927:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204494.056931:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204494.056937:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204494.056942:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5948 00000400:00000200:0.0:1788204494.056948:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204494.056955:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204494.056961:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204494.056968:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056973:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.056978:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.056982:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204494.056985:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204494.056988:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204494.056992:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204494.056996:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204494.057002:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204494.057007:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204494.057015:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057018:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.057022:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057027:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204494.057032:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204494.057037:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057043:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204494.057043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204494.057048:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1788204494.057049:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 02000000:00000001:0.0:1788204494.057053:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204494.057059:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057063:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204494.057068:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204494.057072:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057076:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204494.057082:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204494.057087:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204494.057092:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204494.057095:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057098:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204494.057102:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204494.057106:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204494.057112:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204494.057118:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204494.057127:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205868416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204494.057154:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204494.057157:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204494.057161:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205868416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204494.057176:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204494.057190:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204494.057195:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204494.057197:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204494.057200:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204494.057205:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204494.057210:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5948 00000400:00000200:0.0:1788204494.057216:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204494.057222:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204494.057227:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204494.057233:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057237:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.057242:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057247:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204494.057251:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204494.057254:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204494.057257:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204494.057261:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204494.057264:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204494.057266:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204494.057270:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057274:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204494.057277:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057297:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204494.057302:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204494.057309:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204494.057314:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204494.057319:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204494.057322:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204494.057328:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057332:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204494.057335:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204494.057340:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057344:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204494.057352:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204494.057356:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204494.057395:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204494.057400:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204494.057405:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204494.057409:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204494.057412:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204494.057419:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204494.057426:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204494.057438:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205868544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204494.057456:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204494.057461:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204494.057470:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205868544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204494.057489:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204494.057495:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5959) 00000800:00000200:2.1:1788204494.057681:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204494.057683:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788204494.057946:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204494.057948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1788204494.058211:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204494.058213:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00010000:00000001:1.0:1788204494.248528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204494.248534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204494.248540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204494.248541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204494.248544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204494.248546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204494.248549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204494.248550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204494.248568:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204494.248570:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204494.248579:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204495.016500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204495.016506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204495.016508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204495.016513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204495.016516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204495.016519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204495.016522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204495.016524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204495.016527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204495.080479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204495.080489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204495.080494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204495.080501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204495.080506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204495.080512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204495.080524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204495.080527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204495.080532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204495.272529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204495.272537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204495.272544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204495.272545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204495.272548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204495.272550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204495.272555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204495.272557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204495.272631:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204495.272636:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204495.272639:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204496.040500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204496.040510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204496.040512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204496.040519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204496.040523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204496.040528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204496.040539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204496.040543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204496.040547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204496.104685:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204496.104689:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204496.104690:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204496.104692:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204496.104694:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204496.104695:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204496.104697:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204496.104698:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204496.104700:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204496.296512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204496.296519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204496.296526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204496.296527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204496.296530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204496.296531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204496.296535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204496.296536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204496.296554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204496.296557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204496.296560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204497.064454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204497.064461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204497.064463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204497.064468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204497.064472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204497.064475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204497.064477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204497.064479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204497.064482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204497.128468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204497.128477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204497.128481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204497.128489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204497.128495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204497.128499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204497.128504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204497.128508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204497.128511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204497.320460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204497.320467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204497.320479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204497.320481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204497.320484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204497.320485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204497.320488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204497.320490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204497.320540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204497.320543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204497.320546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204498.088485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204498.088492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204498.088494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204498.088500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204498.088504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204498.088507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204498.088510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204498.088512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204498.088515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204498.152477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204498.152485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204498.152487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204498.152492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204498.152496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204498.152500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204498.152509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204498.152510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204498.152512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204498.344508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204498.344514:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204498.344515:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204498.344517:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344520:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204498.344521:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344522:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344524:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204498.344527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204498.344529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204498.344530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204498.344543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204498.344545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204498.344546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204498.664464:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300625409, at 4300625920 00000100:00000040:1.0:1788204498.664464:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300625409, at 4300625920 00000100:00000010:1.0:1788204498.664479:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000010:0.0:1788204498.664481:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000001:1.0:1788204499.112451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204499.112459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204499.112461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204499.112466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204499.112470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204499.112474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204499.112477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204499.112479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204499.112482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204499.176479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204499.176486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204499.176488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204499.176493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204499.176496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204499.176500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204499.176508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204499.176509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204499.176511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204499.176589:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204499.176593:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204499.176594:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204499.176597:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204499.176599:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204499.176604:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5954 00000400:00000200:0.0:1788204499.176611:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204499.176615:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204499.176618:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204499.176620:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176622:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176624:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176626:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204499.176628:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204499.176630:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204499.176632:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204499.176635:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204499.176636:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204499.176637:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204499.176639:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176640:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176641:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176642:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204499.176644:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204499.176646:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176647:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204499.176649:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204499.176651:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204499.176653:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176654:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204499.176656:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204499.176663:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204499.176666:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204499.176668:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204499.176669:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176671:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204499.176673:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204499.176676:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204499.176682:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204499.176687:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204499.176693:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205868672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204499.176700:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204499.176703:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204499.176706:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205868672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204499.176728:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204499.176730:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204499.176732:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204499.176733:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204499.176734:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204499.176736:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204499.176737:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5954 00000400:00000200:0.0:1788204499.176739:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204499.176741:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204499.176742:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204499.176744:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176745:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176747:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176748:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204499.176749:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204499.176749:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204499.176750:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204499.176752:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204499.176753:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204499.176754:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204499.176756:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176757:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176759:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176760:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204499.176761:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204499.176763:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176764:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204499.176766:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204499.176770:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204499.176771:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176773:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204499.176777:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204499.176779:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176781:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204499.176785:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204499.176789:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204499.176791:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204499.176794:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176795:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204499.176797:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204499.176799:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204499.176801:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204499.176805:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204499.176812:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205868800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204499.176824:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204499.176827:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204499.176830:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205868800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204499.176839:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204499.176841:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204499.176847:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204499.176848:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204499.176852:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204499.176856:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204499.176859:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5954 00000400:00000200:0.0:1788204499.176863:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204499.176866:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204499.176868:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204499.176874:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176876:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176880:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176883:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204499.176885:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204499.176885:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204499.176886:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204499.176888:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204499.176892:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204499.176895:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204499.176901:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176902:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.176904:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176906:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204499.176910:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204499.176912:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204499.176913:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204499.176915:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204499.176918:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204499.176920:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176921:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204499.176923:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204499.176925:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176929:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204499.176932:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204499.176934:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204499.176936:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204499.176937:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.176940:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204499.176942:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204499.176945:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204499.176949:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204499.176959:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204499.176967:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205868928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204499.176985:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204499.176989:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204499.176994:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205868928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204499.177001:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204499.177003:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204499.177007:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204499.177008:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204499.177011:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204499.177015:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204499.177016:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5954 00000400:00000200:0.0:1788204499.177020:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204499.177023:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204499.177026:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204499.177028:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.177029:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.177030:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.177031:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204499.177036:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204499.177038:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204499.177041:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204499.177042:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204499.177044:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204499.177046:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204499.177050:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.177053:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204499.177055:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204499.177057:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204499.177061:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204499.177063:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204499.177066:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204499.177068:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204499.177072:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204499.177076:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.177079:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204499.177083:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204499.177088:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.177092:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204499.177097:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204499.177102:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204499.177107:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204499.177108:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204499.177110:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204499.177113:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204499.177116:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204499.177120:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204499.177125:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204499.177131:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205869056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204499.177142:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204499.177145:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204499.177153:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205869056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204499.177168:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204499.177174:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204499.177175:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5964) 00000800:00000200:2.1:1788204499.177181:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204499.177621:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204499.177624:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204499.177900:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204499.177903:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1788204499.178171:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204499.178173:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00010000:00000001:1.0:1788204499.368476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204499.368481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204499.368486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204499.368487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204499.368489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204499.368490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204499.368492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204499.368493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204499.368561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204499.368564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204499.368568:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204500.136475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204500.136482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204500.136483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204500.136488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204500.136490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204500.136493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204500.136495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204500.136496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204500.136498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204500.200459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204500.200466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204500.200469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204500.200475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204500.200478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204500.200482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204500.200486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204500.200488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204500.200492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204500.392509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204500.392516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204500.392521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204500.392522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204500.392524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204500.392525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204500.392526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204500.392527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204500.392541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204500.392543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204500.392546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204501.160496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204501.160503:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204501.160505:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204501.160510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204501.160513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204501.160517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204501.160518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204501.160520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204501.160522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204501.224483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204501.224492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204501.224496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204501.224505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204501.224510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204501.224514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204501.224517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204501.224521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204501.224523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204501.416480:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204501.416484:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204501.416493:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416498:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204501.416501:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416506:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204501.416512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204501.416518:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204501.416519:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204501.416521:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416522:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204501.416524:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416525:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416527:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204501.416533:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204501.416534:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204501.416535:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416537:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204501.416538:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416540:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204501.416542:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204501.416603:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204501.416608:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204501.416612:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204502.184518:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204502.184526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204502.184528:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204502.184533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204502.184537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204502.184541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204502.184543:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204502.184544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204502.184547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204502.248475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204502.248482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204502.248486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204502.248491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204502.248497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204502.248502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204502.248508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204502.248511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204502.248515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204502.440513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204502.440518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204502.440523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204502.440524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204502.440526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204502.440527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204502.440528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204502.440529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204502.440543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204502.440545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204502.440547:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204503.208504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204503.208513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204503.208515:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204503.208521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204503.208525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204503.208528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204503.208531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204503.208533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204503.208536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204503.272632:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204503.272635:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204503.272636:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204503.272638:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204503.272640:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204503.272650:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204503.272652:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204503.272653:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204503.272655:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204503.464457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204503.464464:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204503.464471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204503.464473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204503.464476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204503.464477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204503.464480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204503.464481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204503.464528:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204503.464532:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204503.464535:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204503.784447:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300630529, at 4300631040 00000100:00000040:1.0:1788204503.784448:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300630529, at 4300631040 00000100:00000010:1.0:1788204503.784458:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000010:0.0:1788204503.784470:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00000001:1.0:1788204504.232467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204504.232475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204504.232477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204504.232483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204504.232486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204504.232491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204504.232493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204504.232495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204504.232498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204504.296480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204504.296487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204504.296492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204504.296497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204504.296503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204504.296508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204504.296514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204504.296518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204504.296521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204504.297034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204504.297039:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204504.297538:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204504.297541:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204504.297784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204504.297786:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204504.297994:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204504.297996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00010000:00000001:1.0:1788204504.488494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204504.488499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204504.488504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204504.488505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204504.488507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204504.488508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204504.488510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204504.488510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204504.488523:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204504.488525:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204504.488527:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204505.256496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204505.256513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204505.256516:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204505.256521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204505.256525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204505.256528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204505.256531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204505.256533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204505.256536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204505.320463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204505.320471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204505.320475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204505.320482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204505.320489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204505.320493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204505.320495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204505.320506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204505.320510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204505.512468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204505.512475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204505.512482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204505.512484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204505.512487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204505.512488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204505.512492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204505.512493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204505.512565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204505.512569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204505.512573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204506.280491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204506.280498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204506.280499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204506.280505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204506.280508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204506.280512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204506.280515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204506.280516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204506.280518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204506.344465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204506.344472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204506.344476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204506.344483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204506.344489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204506.344497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204506.344504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204506.344507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204506.344510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204506.536496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204506.536510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204506.536516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204506.536518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204506.536521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204506.536522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204506.536525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204506.536527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204506.536544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204506.536546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204506.536548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204507.304533:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204507.304544:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204507.304547:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204507.304554:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204507.304559:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204507.304565:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204507.304568:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204507.304571:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204507.304574:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204507.368501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204507.368512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204507.368514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204507.368521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204507.368526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204507.368533:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204507.368536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204507.368539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204507.368544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204507.560471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204507.560479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204507.560484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204507.560486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204507.560489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204507.560490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204507.560494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204507.560495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204507.560546:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204507.560550:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204507.560553:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204508.328485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204508.328491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204508.328493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204508.328498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204508.328500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204508.328504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204508.328505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204508.328507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204508.328509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204508.392496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204508.392507:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204508.392511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204508.392518:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204508.392524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204508.392531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204508.392543:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204508.392548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204508.392551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204508.584532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204508.584540:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204508.584543:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204508.584546:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584549:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204508.584550:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584552:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584554:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204508.584557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204508.584559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204508.584560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204508.584591:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204508.584594:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204508.584596:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204508.904511:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300635649, at 4300636160 00000100:00000040:0.0:1788204508.904517:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300635649, at 4300636160 00000100:00000010:1.0:1788204508.904534:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00000010:0.0:1788204508.904538:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000001:1.0:1788204509.352493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204509.352502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204509.352504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204509.352510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204509.352513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204509.352517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204509.352519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204509.352522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204509.352525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204509.416443:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204509.416449:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204509.416450:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204509.416453:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204509.416456:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204509.416458:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204509.416460:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204509.416462:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204509.416464:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204509.416558:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204509.416564:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204509.416568:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204509.416572:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204509.416577:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204509.416584:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5964 00000400:00000200:0.0:1788204509.416592:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204509.416597:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204509.416600:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204509.416603:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416606:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.416609:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416611:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204509.416614:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204509.416616:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204509.416618:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204509.416622:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204509.416624:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204509.416626:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204509.416629:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416631:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.416632:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416634:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204509.416646:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204509.416652:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416655:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204509.416658:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204509.416661:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204509.416663:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.416665:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204509.416668:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204509.416676:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204509.416679:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204509.416683:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204509.416685:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.416688:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204509.416690:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204509.416694:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204509.416702:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204509.416708:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204509.416716:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205869696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204509.416724:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204509.416728:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204509.416733:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205869696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.416751:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204509.416754:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204509.416758:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204509.416759:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204509.416761:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204509.416763:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204509.416766:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5964 00000400:00000200:0.0:1788204509.416769:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204509.416772:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204509.416774:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204509.416778:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416781:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.416783:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416785:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204509.416787:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204509.416789:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204509.416791:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204509.416794:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204509.416796:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204509.416799:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204509.416802:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416804:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.416807:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416808:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204509.416811:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204509.416813:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204509.416815:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204509.416817:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204509.416822:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204509.416825:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.416830:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204509.416834:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204509.416840:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.416844:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204509.416849:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204509.416853:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204509.416861:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204509.416864:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.416869:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204509.416872:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204509.416877:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204509.416885:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204509.416893:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204509.416905:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205869824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204509.416934:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204509.416941:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204509.416951:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205869824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.416967:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204509.416973:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204509.416984:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204509.416986:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204509.416988:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204509.416991:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204509.416994:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5964 00000400:00000200:0.0:1788204509.417004:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204509.417011:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204509.417028:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204509.417032:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417034:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.417036:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417039:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204509.417042:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204509.417043:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204509.417046:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204509.417050:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204509.417054:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204509.417058:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204509.417070:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417078:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.417088:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417096:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204509.417105:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204509.417114:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417119:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204509.417126:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204509.417131:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204509.417137:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417140:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204509.417146:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204509.417155:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417161:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204509.417167:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204509.417170:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204509.417181:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204509.417182:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417189:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204509.417193:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204509.417197:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204509.417204:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204509.417211:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204509.417222:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205869952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204509.417271:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204509.417277:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204509.417291:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205869952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.417317:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204509.417321:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204509.417330:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204509.417332:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204509.417337:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204509.417345:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.1:1788204509.417351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204509.417353:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5964 00000400:00000200:0.0:1788204509.417417:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204509.417424:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204509.417436:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204509.417450:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417458:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.417469:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417474:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204509.417479:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204509.417481:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204509.417484:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204509.417489:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204509.417494:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204509.417499:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204509.417511:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417522:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204509.417529:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417533:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204509.417539:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204509.417546:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204509.417551:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204509.417555:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204509.417561:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204509.417568:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417575:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204509.417580:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204509.417591:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417600:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204509.417612:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204509.417619:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204509.417626:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204509.417632:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.417644:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204509.417647:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204509.417654:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204509.417662:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204509.417670:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204509.417689:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205870080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204509.417708:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204509.417715:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204509.417726:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205870080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.417758:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204509.417769:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5974) 00000800:00000200:2.1:1788204509.417834:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204509.417964:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204509.418120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00010000:00000001:1.0:1788204509.608510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204509.608516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204509.608522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204509.608523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204509.608526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204509.608527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204509.608530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204509.608531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204509.608591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204509.608595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204509.608599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204510.376533:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204510.376542:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204510.376543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204510.376547:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204510.376550:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204510.376553:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204510.376555:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204510.376556:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204510.376558:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204510.440475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204510.440487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204510.440490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204510.440497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204510.440501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204510.440506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204510.440509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204510.440512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204510.440515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204510.632487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204510.632494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204510.632499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204510.632500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204510.632504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204510.632505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204510.632508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204510.632509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204510.632526:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204510.632529:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204510.632532:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204511.400502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204511.400510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204511.400511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204511.400515:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204511.400518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204511.400522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204511.400523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204511.400525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204511.400528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204511.464490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204511.464495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204511.464497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204511.464501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204511.464504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204511.464507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204511.464509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204511.464511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204511.464514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204511.656520:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204511.656523:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204511.656527:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656533:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204511.656535:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656537:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656539:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204511.656545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204511.656550:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204511.656551:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204511.656552:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656554:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204511.656555:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656556:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656563:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204511.656568:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204511.656569:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204511.656571:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656572:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204511.656574:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656575:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204511.656577:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204511.656652:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204511.656655:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204511.656658:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204512.424473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204512.424478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204512.424479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204512.424484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204512.424487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204512.424490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204512.424491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204512.424493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204512.424495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204512.488448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204512.488456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204512.488458:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204512.488463:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204512.488467:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204512.488471:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204512.488474:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204512.488477:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204512.488480:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204512.680547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204512.680555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204512.680560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204512.680561:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204512.680564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204512.680566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204512.680570:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204512.680571:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204512.680591:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204512.680595:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204512.680598:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204513.448496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204513.448505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204513.448507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204513.448513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204513.448516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204513.448520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204513.448522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204513.448524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204513.448526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204513.512460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204513.512466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204513.512468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204513.512472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204513.512475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204513.512478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204513.512480:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204513.512482:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204513.512484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204513.704501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204513.704508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204513.704517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204513.704519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204513.704523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204513.704524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204513.704528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204513.704530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204513.704877:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204513.704882:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204513.704885:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204514.024454:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300640770, at 4300641280 00000100:00000040:0.0:1788204514.024467:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300640770, at 4300641280 00000100:00000010:1.0:1788204514.024471:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000010:0.0:1788204514.024477:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000001:1.0:1788204514.472473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204514.472480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204514.472481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204514.472486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204514.472488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204514.472492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204514.472494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204514.472496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204514.472498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204514.536570:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204514.536578:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204514.536580:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204514.536585:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204514.536588:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204514.536592:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204514.536594:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204514.536597:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204514.536600:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204514.537486:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204514.537493:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204514.538002:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204514.538021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000010:2.0:1788204514.538028:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204514.538032:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204514.538036:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204514.538040:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204514.538042:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204514.538044:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204514.538048:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538056:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538059:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538066:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538067:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204514.538070:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204514.538074:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204514.538077:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204514.538080:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300641947 00000100:00000001:2.0:1788204514.538082:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538085:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300641947 00000100:00000001:2.0:1788204514.538087:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538089:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204514.538091:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538093:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204514.538097:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204514/real 1788204514] req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204514.538105:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204514.538106:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204514.538108:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204514.538111:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204514.538112:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204514.538114:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204514.538118:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204514.538125:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538131:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204514.538132:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538134:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538135:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204514.538138:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538144:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204514.538148:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538155:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204514.538157:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538159:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538160:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538161:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204514.538165:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204514.538168:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204514.538171:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204514.538174:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204514.538178:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204514.538185:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870336:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204514.538188:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204514.538191:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538202:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204514.538204:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204514.538206:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204514.538208:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204514.538209:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204514.538213:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870464:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204514.538220:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204514.538222:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204514.538224:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204514.538226:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204514.538229:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204514.538231:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204514.538235:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204514.538237:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204514.538241:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204514.538243:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205870464, portal 4 00000100:00000001:2.0:1788204514.538244:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204514.538246:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:2.0:1788204514.538252:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538257:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204514.538259:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204514.538262:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204514.538264:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205870464, offset 0 00000400:00000200:2.0:1788204514.538268:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204514.538287:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204514.538291:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204514.538294:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204514.538296:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204514.538299:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204514.538301:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204514.538306:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58656:58656:256:4294967295] 192.168.201.151@tcp LPNI seq info [58636:58636:8:4294967295] 00000400:00000200:2.0:1788204514.538309:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204514.538315:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204514.538320:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204514.538323:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204514.538326:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204514.538329:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204514.538333:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204514.538335:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204514.538339:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204514.538341:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204514.538344:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204514.538347:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204514.538349:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204514.538351:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204514.538353:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204514.538357:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538389:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538392:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538399:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538400:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204514.538403:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204514.538407:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204514.538409:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204514.538411:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300641834 00000100:00000001:2.0:1788204514.538414:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538415:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300641834 00000100:00000001:2.0:1788204514.538417:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538420:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204514.538422:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538424:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204514.538427:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204514/real 1788204514] req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204514.538435:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204514.538436:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204514.538438:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204514.538441:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204514.538443:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204514.538444:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204514.538447:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204514.538455:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538460:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204514.538462:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538464:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538465:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204514.538468:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538474:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204514.538477:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538484:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204514.538486:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538488:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538489:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538490:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204514.538492:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204514.538495:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204514.538498:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204514.538500:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204514.538504:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204514.538511:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870464:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204514.538515:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538516:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538519:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205870336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538525:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204514.538527:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204514.538529:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204514.538531:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204514.538533:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204514.538534:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204514.538536:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204514.538538:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:2.0:1788204514.538540:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204514.538541:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204514.538542:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:2.0:1788204514.538544:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204514.538545:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538547:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538550:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205870464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538555:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204514.538557:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204514.538558:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204514.538560:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204514.538561:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204514.538562:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204514.538564:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204514.538565:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:2.0:1788204514.538567:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204514.538568:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204514.538569:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:2.0:1788204514.538571:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204514.538572:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538573:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538577:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204514.538579:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538581:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204514.538582:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204514.538584:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204514.538588:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538593:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204514.538594:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204514.538596:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204514.538598:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204514.538600:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204514.538603:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204514.538606:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204514.538608:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204514.538609:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204514.538611:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204514.538614:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204514.538615:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204514.538619:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204514.538621:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204514.538624:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204514.538627:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205870592, portal 4 00000100:00000001:2.0:1788204514.538629:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204514.538630:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000040:2.0:1788204514.538635:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538645:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204514.538647:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204514.538648:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204514.538651:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205870592, offset 0 00000400:00000200:2.0:1788204514.538654:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204514.538658:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204514.538661:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204514.538664:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204514.538666:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204514.538668:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204514.538670:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204514.538673:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58657:58657:256:4294967295] 192.168.201.151@tcp LPNI seq info [58637:58637:8:4294967295] 00000400:00000200:2.0:1788204514.538677:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204514.538683:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204514.538687:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204514.538690:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204514.538694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204514.538697:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204514.538701:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204514.538703:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204514.538706:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204514.538708:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204514.538710:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204514.538713:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204514.538716:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204514.538718:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204514.538720:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204514.538723:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538731:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538734:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538746:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538747:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204514.538756:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204514.538759:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204514.538763:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204514.538765:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300641840 00000100:00000001:2.0:1788204514.538767:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538769:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300641840 00000100:00000001:2.0:1788204514.538771:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300645890 00000100:00000001:2.0:1788204514.538773:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204514.538775:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538777:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204514.538780:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204514/real 1788204514] req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204514.538788:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204514.538789:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204514.538791:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204514.538793:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204514.538795:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204514.538797:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204514.538800:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204514.538808:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538814:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204514.538816:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538817:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538819:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204514.538822:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538829:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204514.538832:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538838:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204514.538840:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538841:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204514.538842:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538844:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204514.538846:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204514.538849:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204514.538852:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204514.538853:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204514.538857:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204514.538864:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204514.538868:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538870:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204514.538873:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205870592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204514.538879:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204514.538880:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204514.538882:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204514.538885:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204514.538886:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204514.538888:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204514.538890:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204514.538892:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:2.0:1788204514.538894:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204514.538895:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204514.538896:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204514.538898:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204514.538899:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538901:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204514.538904:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204514.538905:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204514.538907:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204514.538908:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204514.728492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204514.728497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204514.728500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204514.728502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204514.728504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204514.728505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204514.728507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204514.728508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204514.728520:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204514.728522:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204514.728524:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204515.048464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204515.048471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204515.048473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204515.048477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204515.048481:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204515.048485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204515.048488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204515.048491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204515.048494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204515.496464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204515.496476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204515.496477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204515.496481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204515.496484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204515.496486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204515.496488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204515.496490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204515.496491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204515.560451:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204515.560451:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204515.560459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204515.560461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204515.560465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204515.560468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204515.560469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204515.560473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204515.560478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204515.560479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204515.560482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204515.560484:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204515.560486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204515.560488:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204515.560493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204515.560494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204515.560496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204515.560500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204515.752458:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204515.752465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204515.752471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204515.752473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204515.752477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204515.752478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204515.752481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204515.752482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204515.752534:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204515.752537:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204515.752540:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204516.072471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204516.072477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204516.072478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204516.072482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204516.072485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204516.072489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204516.072491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204516.072494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204516.072496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204516.520456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204516.520463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204516.520465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204516.520470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204516.520474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204516.520478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204516.520480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204516.520482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204516.520484:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204516.584452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204516.584460:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204516.584462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204516.584467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204516.584471:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204516.584474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204516.584477:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204516.584480:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204516.584483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204516.776503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204516.776515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204516.776521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204516.776522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204516.776525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204516.776526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204516.776529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204516.776530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204516.776547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204516.776549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204516.776552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204517.096473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204517.096481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204517.096483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204517.096488:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204517.096492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204517.096495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204517.096498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204517.096501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204517.096504:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204517.544480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204517.544487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204517.544490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204517.544495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204517.544499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204517.544503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204517.544505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204517.544508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204517.544511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204517.608590:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204517.608592:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204517.608593:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204517.608595:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204517.608596:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204517.608598:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204517.608599:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204517.608600:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204517.608601:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204517.800462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204517.800477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204517.800483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204517.800485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204517.800488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204517.800489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204517.800492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204517.800493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204517.800551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204517.800555:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204517.800558:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204518.120451:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204518.120458:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204518.120460:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204518.120465:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204518.120468:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204518.120472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204518.120479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204518.120482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204518.120484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204518.568464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204518.568472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204518.568474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204518.568479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204518.568483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204518.568487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204518.568489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204518.568492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204518.568495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204518.632473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204518.632481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204518.632485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204518.632491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204518.632497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204518.632502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204518.632514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204518.632518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204518.632521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204518.824526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204518.824534:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204518.824536:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204518.824540:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824543:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204518.824546:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824548:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824550:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204518.824554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204518.824558:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204518.824559:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204518.824575:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204518.824577:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204518.824580:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204519.144482:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300645890, at 4300646400 00000100:00000040:1.0:1788204519.144482:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300645890, at 4300646400 00000100:00000010:0.0:1788204519.144498:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000010:1.0:1788204519.144500:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000001:2.0:1788204519.144603:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204519.144607:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204519.144609:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204519.144613:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204519.144615:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204519.144618:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.144621:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204519.144624:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.144626:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204519.592476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204519.592483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204519.592486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204519.592491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204519.592495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204519.592498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204519.592499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204519.592501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204519.592503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204519.656482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204519.656492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204519.656495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204519.656499:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204519.656504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204519.656510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204519.656511:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204519.656516:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204519.656518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204519.656520:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204519.656523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204519.656526:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204519.656528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204519.656532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204519.656534:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5974 00000400:00000200:0.0:1788204519.656545:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204519.656552:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204519.656555:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204519.656559:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656563:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204519.656566:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656569:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204519.656572:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204519.656575:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204519.656577:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204519.656580:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204519.656583:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204519.656585:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204519.656588:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656589:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204519.656591:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656593:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204519.656596:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204519.656598:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656601:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204519.656603:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204519.656606:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204519.656609:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.656611:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204519.656613:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204519.656621:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204519.656624:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204519.656627:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204519.656629:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.656632:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204519.656634:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204519.656637:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204519.656648:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204519.656655:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204519.656664:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204519.656673:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204519.656677:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204519.656681:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204519.656701:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204519.656704:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204519.656707:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204519.656709:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204519.656711:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204519.656714:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204519.656716:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5974 00000400:00000200:0.0:1788204519.656720:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204519.656723:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204519.656726:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204519.656730:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656733:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204519.656735:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656737:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204519.656740:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204519.656742:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:2.0:1788204519.656742:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204519.656747:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204519.656747:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788204519.656750:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204519.656751:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204519.656754:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204519.656758:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204519.656760:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204519.656764:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656771:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204519.656775:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1788204519.656776:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656780:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204519.656782:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204519.656783:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204519.656788:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788204519.656789:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:2.0:1788204519.656792:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204519.656795:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.656797:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204519.656800:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788204519.656803:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205870720:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204519.656807:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204519.656809:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204519.656812:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.656815:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204519.656816:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204519.656818:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204519.656822:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204519.656822:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788204519.656826:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.656828:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204519.656830:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 00000100:00000010:0.0:1788204519.656832:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 02000000:00000001:2.0:1788204519.656833:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204519.656847:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204519.656850:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204519.656851:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.656852:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204519.656853:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204519.656854:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204519.656857:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204519.656859:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204519.656865:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204519.656868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204519.656875:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00000001:0.0:1788204519.656877:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204519.656881:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204519.656883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000040:0.0:1788204519.656886:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204519.656891:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205870720, portal 25 00000100:00000001:2.0:1788204519.656895:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204519.656898:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204519.656902:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.656908:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204519.656912:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204519.656917:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204519.656919:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204519.656920:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204519.656924:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1788204519.656925:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204519.656930:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:2.0:1788204519.656931:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00080000:0.0:1788204519.656936:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5974 00000100:00000200:2.0:1788204519.656936:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205870720, offset 0 00000400:00000200:0.0:1788204519.656943:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.656948:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204519.656952:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204519.656955:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204519.656960:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.656961:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204519.656966:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204519.656970:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.656972:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204519.656973:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204519.656977:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204519.656979:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204519.656981:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204519.656981:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1788204519.656985:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:2.0:1788204519.656987:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788204519.656988:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204519.656991:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204519.656994:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204519.656996:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657000:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204519.657000:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:0.0:1788204519.657005:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657009:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58658:58658:256:4294967295] 192.168.201.151@tcp LPNI seq info [58638:58638:8:4294967295] 00000100:00000001:0.0:1788204519.657011:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204519.657018:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204519.657019:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204519.657024:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657028:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204519.657034:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204519.657034:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204519.657040:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204519.657045:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1788204519.657045:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 02000000:00000001:0.0:1788204519.657049:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.657051:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.0:1788204519.657054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00100000:0.0:1788204519.657055:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204519.657060:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204519.657060:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204519.657061:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204519.657068:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000800:00000200:2.1:1788204519.657068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204519.657071:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.1:1788204519.657073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1788204519.657076:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204519.657080:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788204519.657080:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657084:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:2.0:1788204519.657084:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 02000000:00000001:0.0:1788204519.657087:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204519.657089:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204519.657090:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204519.657096:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000010:0.0:1788204519.657097:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000400:00000200:2.0:1788204519.657099:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000100:00000001:0.0:1788204519.657102:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204519.657102:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204519.657106:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1788204519.657111:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1788204519.657112:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204519.657140:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204519.657140:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204519.657145:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.657146:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657155:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657157:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1788204519.657158:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657163:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204519.657169:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204519.657171:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657172:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300651009 00000100:00080000:0.0:1788204519.657176:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:2.0:1788204519.657178:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c10c0. 00000100:00080000:0.0:1788204519.657181:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204519.657184:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204519.657186:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204519.657186:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204519.657192:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204519.657193:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300651009 00000100:00000010:2.0:1788204519.657195:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1960. 00000100:00000001:2.0:1788204519.657198:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788204519.657199:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5974 00000100:00000001:2.0:1788204519.657201:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204519.657204:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204519.657207:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204519.657209:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:0.0:1788204519.657211:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204519.657215:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788204519.657218:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204519/real 1788204519] req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204519.657221:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657226:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204519.657231:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657236:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000001:2.0:1788204519.657236:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204519.657240:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1788204519.657241:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:2.0:1788204519.657244:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000020:00000040:0.0:1788204519.657245:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204519.657248:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204519.657251:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204519.657253:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204519.657255:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204519.657257:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000100:00000040:0.0:1788204519.657259:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:2.0:1788204519.657260:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204519.657264:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204519.657265:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1788204519.657267:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657270:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204519.657274:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000200:2.0:1788204519.657274:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204519.657279:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204519.657283:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204519.657287:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204519.657290:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204519.657293:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.657295:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:2.0:1788204519.657295:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788204519.657298:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204519.657303:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.657306:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204519.657312:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204519.657313:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1788204519.657316:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204519.657317:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204519.657319:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204519.657321:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788204519.657322:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204519.657325:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204519.657326:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204519.657327:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204519.657329:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204519.657333:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204519.657334:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204519.657338:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204519.657346:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204519.657347:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204519.657354:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204519.657357:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.657358:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204519.657361:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204519.657395:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657400:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1788204519.657401:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5984) 00000100:00000001:2.0:1788204519.657405:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657408:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.657410:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657413:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204519.657415:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204519.657419:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204519.657424:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204519.657426:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204519.657431:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204519.657445:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205870720:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204519.657452:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657454:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.657457:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205870720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657464:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204519.657466:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204519.657468:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204519.657472:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204519.657473:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204519.657475:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204519.657477:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204519.657480:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:2.0:1788204519.657483:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204519.657485:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204519.657486:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204519.657489:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204519.657491:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657493:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657499:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204519.657500:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657502:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204519.657503:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204519.657504:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204519.657506:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657511:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204519.657512:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657513:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204519.657514:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204519.657515:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204519.657517:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870848:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204519.657519:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204519.657520:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204519.657521:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657523:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204519.657525:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204519.657526:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204519.657529:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204519.657531:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204519.657533:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204519.657535:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205870848, portal 10 00000100:00000001:2.0:1788204519.657536:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204519.657538:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204519.657542:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657546:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204519.657547:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204519.657548:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204519.657550:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205870848, offset 0 00000400:00000200:2.0:1788204519.657552:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657554:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657556:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204519.657559:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204519.657560:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204519.657562:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204519.657563:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657565:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58659:58659:256:4294967295] 192.168.201.151@tcp LPNI seq info [58639:58639:8:4294967295] 00000400:00000200:2.0:1788204519.657567:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657571:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204519.657575:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657578:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204519.657582:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204519.657584:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204519.657588:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204519.657590:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204519.657593:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657595:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204519.657598:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204519.657602:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204519.657604:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204519.657606:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204519.657608:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204519.657612:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657619:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.657622:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657627:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657628:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204519.657631:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.657633:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204519.657635:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204519.657637:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300647067 00000100:00000001:2.0:1788204519.657639:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.657641:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300647067 00000100:00000001:2.0:1788204519.657643:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.657645:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204519.657647:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657649:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204519.657652:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204519/real 1788204519] req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204519.657660:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204519.657661:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204519.657662:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204519.657665:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204519.657666:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204519.657668:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204519.657672:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204519.657680:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657684:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204519.657685:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657686:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.657687:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204519.657689:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657693:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204519.657695:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657699:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204519.657700:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657702:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.657702:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657703:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204519.657705:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204519.657707:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204519.657708:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204519.657709:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204519.657711:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204519.657716:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870848:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204519.657718:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204519.657720:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657728:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204519.657729:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657730:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204519.657731:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204519.657733:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204519.657734:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870976:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204519.657736:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204519.657737:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204519.657738:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657739:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204519.657741:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204519.657742:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204519.657744:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204519.657746:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204519.657748:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204519.657750:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205870976, portal 4 00000100:00000001:2.0:1788204519.657751:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204519.657752:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204519.657755:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657759:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204519.657761:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204519.657762:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204519.657763:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205870976, offset 0 00000400:00000200:2.0:1788204519.657765:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657767:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657769:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204519.657771:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204519.657773:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204519.657774:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204519.657775:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657778:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58660:58660:256:4294967295] 192.168.201.151@tcp LPNI seq info [58640:58640:8:4294967295] 00000400:00000200:2.0:1788204519.657780:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657784:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204519.657787:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657788:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204519.657791:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204519.657792:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204519.657795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204519.657796:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204519.657798:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657799:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204519.657801:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204519.657803:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204519.657805:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204519.657806:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204519.657807:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204519.657809:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657814:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.657815:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657819:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657820:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204519.657821:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.657823:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204519.657825:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204519.657826:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300646953 00000100:00000001:2.0:1788204519.657827:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.657829:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300646953 00000100:00000001:2.0:1788204519.657830:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.657831:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204519.657832:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657833:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204519.657848:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204519/real 1788204519] req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204519.657854:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204519.657855:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204519.657856:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204519.657858:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204519.657859:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204519.657860:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204519.657862:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204519.657867:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657871:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204519.657872:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657873:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.657873:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204519.657875:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657878:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204519.657880:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657884:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204519.657885:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657886:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.657887:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.657888:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204519.657889:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204519.657891:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204519.657893:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204519.657894:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204519.657896:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204519.657901:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205870976:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204519.657902:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204519.657904:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657907:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204519.657908:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657909:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204519.657910:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204519.657911:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204519.657913:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204519.657915:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204519.657916:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204519.657917:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204519.657918:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204519.657920:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204519.657921:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204519.657922:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204519.657924:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204519.657925:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204519.657927:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205871104, portal 4 00000100:00000001:2.0:1788204519.657928:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204519.657929:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204519.657932:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657936:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204519.657938:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204519.657939:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204519.657940:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205871104, offset 0 00000400:00000200:2.0:1788204519.657942:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657945:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657947:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204519.657948:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204519.657950:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204519.657952:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204519.657953:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204519.657955:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58661:58661:256:4294967295] 192.168.201.151@tcp LPNI seq info [58641:58641:8:4294967295] 00000400:00000200:2.0:1788204519.657957:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204519.657961:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204519.657963:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657965:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204519.657967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204519.657969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204519.657971:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204519.657973:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204519.657975:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204519.657976:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204519.657978:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204519.657980:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204519.657981:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204519.657983:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204519.657984:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204519.657986:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657990:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.657992:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.657995:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.657996:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204519.657998:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204519.658000:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204519.658002:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204519.658003:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300646959 00000100:00000001:2.0:1788204519.658004:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.658006:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300646959 00000100:00000001:2.0:1788204519.658007:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300651009 00000100:00000001:2.0:1788204519.658008:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204519.658009:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.658010:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204519.658011:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204519/real 1788204519] req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204519.658015:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204519.658016:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204519.658017:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204519.658018:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204519.658019:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204519.658020:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204519.658022:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204519.658029:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658035:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204519.658036:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658038:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.658039:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204519.658042:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658049:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204519.658052:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658059:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204519.658062:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658063:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204519.658065:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658066:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204519.658068:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204519.658070:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204519.658074:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204519.658075:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204519.658078:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204519.658083:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204519.658085:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658086:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.658088:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205870848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658092:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204519.658093:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204519.658094:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204519.658096:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204519.658096:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204519.658097:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204519.658099:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204519.658100:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:2.0:1788204519.658101:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204519.658102:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204519.658103:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204519.658105:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204519.658106:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658107:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.658109:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205870976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658112:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204519.658113:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204519.658114:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204519.658115:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204519.658116:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204519.658116:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204519.658117:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204519.658118:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:2.0:1788204519.658119:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204519.658120:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204519.658120:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204519.658121:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204519.658122:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658123:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204519.658124:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205871104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204519.658127:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204519.658128:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204519.658129:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204519.658130:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204519.658130:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204519.658131:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204519.658132:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204519.658132:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:2.0:1788204519.658134:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204519.658135:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204519.658135:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204519.658136:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204519.658137:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658138:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204519.658140:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204519.658141:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204519.658142:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204519.658143:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204519.848480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204519.848486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204519.848491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204519.848493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204519.848499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204519.848501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204519.848504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204519.848506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204519.848554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204519.848558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204519.848560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204520.168470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204520.168477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204520.168479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204520.168485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204520.168489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204520.168493:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204520.168496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204520.168498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204520.168501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204520.616443:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204520.616451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204520.616453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204520.616458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204520.616461:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204520.616465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204520.616468:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204520.616470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204520.616472:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204520.680505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204520.680516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204520.680521:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204520.680532:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204520.680550:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204520.680556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204520.680559:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204520.680562:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204520.680565:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204520.872528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204520.872535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204520.872541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204520.872542:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204520.872545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204520.872546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204520.872550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204520.872551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204520.872574:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204520.872576:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204520.872579:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204521.192475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204521.192483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204521.192485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204521.192491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204521.192495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204521.192498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204521.192500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204521.192503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204521.192505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204521.640500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204521.640511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204521.640513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204521.640520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204521.640523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204521.640527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204521.640530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204521.640532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204521.640535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204521.704501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204521.704521:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204521.704525:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204521.704531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204521.704535:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204521.704540:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204521.704546:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204521.704550:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204521.704555:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204521.896499:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204521.896502:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204521.896505:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896509:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204521.896511:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896513:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896515:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204521.896520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204521.896523:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204521.896524:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204521.896525:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896526:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204521.896526:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896527:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896528:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204521.896532:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204521.896533:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204521.896533:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896541:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204521.896542:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896543:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204521.896544:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204521.896598:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204521.896602:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204521.896606:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204522.216522:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204522.216533:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204522.216536:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204522.216543:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204522.216548:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204522.216553:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204522.216556:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204522.216560:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204522.216563:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204522.664498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204522.664507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204522.664509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204522.664514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204522.664517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204522.664521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204522.664523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204522.664525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204522.664527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204522.728508:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204522.728518:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204522.728521:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204522.728529:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204522.728535:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204522.728540:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204522.728552:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204522.728556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204522.728559:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204522.920610:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204522.920618:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204522.920622:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204522.920623:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204522.920625:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204522.920626:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204522.920628:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204522.920629:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204522.920651:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204522.920653:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204522.920656:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204523.240463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204523.240474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204523.240476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204523.240482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204523.240485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204523.240489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204523.240492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204523.240495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204523.240497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204523.688448:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204523.688456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204523.688458:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204523.688463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204523.688466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204523.688469:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204523.688471:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204523.688474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204523.688476:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204523.752511:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204523.752522:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204523.752529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204523.752537:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204523.752545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204523.752554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204523.752558:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204523.752561:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204523.752565:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204523.944491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204523.944498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204523.944503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204523.944504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204523.944508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204523.944510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204523.944514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204523.944516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204523.944573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204523.944578:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204523.944580:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204524.264516:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300651009, at 4300651520 00000100:00000040:1.0:1788204524.264516:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300651009, at 4300651520 00000100:00000001:2.0:1788204524.264516:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204524.264527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204524.264532:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204524.264540:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204524.264541:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00000001:2.0:1788204524.264544:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204524.264545:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00000001:2.0:1788204524.264552:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.264557:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204524.264561:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.264565:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204524.712503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204524.712511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204524.712512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204524.712517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204524.712520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204524.712524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204524.712527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204524.712529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204524.712532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204524.776500:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204524.776501:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204524.776506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204524.776509:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204524.776510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776514:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204524.776516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204524.776520:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204524.776521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204524.776527:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204524.776527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204524.776537:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5979 00000100:00000001:1.0:1788204524.776541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204524.776544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204524.776547:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204524.776549:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204524.776557:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204524.776561:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204524.776565:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776568:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776571:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776574:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204524.776578:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204524.776580:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204524.776583:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204524.776586:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204524.776588:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204524.776591:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204524.776594:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776596:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776598:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776600:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204524.776603:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204524.776606:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776608:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204524.776611:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204524.776614:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204524.776616:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776619:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204524.776621:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204524.776629:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204524.776633:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204524.776636:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204524.776638:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776641:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204524.776643:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204524.776647:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204524.776655:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204524.776662:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204524.776672:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204524.776682:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204524.776686:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204524.776691:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204524.776721:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204524.776724:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204524.776727:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204524.776728:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204524.776730:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204524.776732:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204524.776735:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5979 00000400:00000200:0.0:1788204524.776739:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204524.776742:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204524.776744:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204524.776760:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776763:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776765:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776767:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204524.776770:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204524.776771:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204524.776773:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204524.776775:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204524.776777:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204524.776779:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204524.776782:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776783:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776785:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776787:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204524.776789:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204524.776792:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776794:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204524.776796:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204524.776800:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204524.776803:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776804:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204524.776808:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204524.776812:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776813:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204524.776817:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204524.776820:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204524.776822:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204524.776824:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776825:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204524.776826:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204524.776828:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204524.776832:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204524.776835:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204524.776840:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204524.776847:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204524.776850:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204524.776853:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204524.776861:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204524.776863:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204524.776866:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204524.776868:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204524.776869:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204524.776872:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204524.776874:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5979 00000400:00000200:0.0:1788204524.776878:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204524.776880:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204524.776882:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204524.776885:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776887:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776888:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776890:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204524.776892:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204524.776893:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204524.776894:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204524.776897:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204524.776898:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204524.776900:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204524.776903:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776905:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776906:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776908:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204524.776910:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204524.776912:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776915:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204524.776917:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204524.776919:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204524.776921:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776923:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204524.776925:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204524.776927:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776929:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204524.776932:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204524.776934:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204524.776936:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204524.776937:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.776939:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204524.776940:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204524.776942:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204524.776945:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204524.776948:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204524.776953:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204524.776959:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204524.776961:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204524.776965:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204524.776972:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204524.776974:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204524.776977:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204524.776978:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204524.776980:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204524.776982:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204524.776985:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5979 00000400:00000200:0.0:1788204524.776988:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204524.776991:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204524.776993:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204524.776996:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.776998:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.776999:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.777001:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204524.777003:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204524.777004:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204524.777005:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204524.777008:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204524.777010:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204524.777012:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204524.777014:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.777017:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204524.777019:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204524.777020:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204524.777023:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204524.777025:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204524.777028:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204524.777030:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204524.777032:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204524.777034:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.777036:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204524.777038:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204524.777040:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.777042:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204524.777045:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204524.777047:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204524.777049:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204524.777050:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204524.777052:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204524.777053:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204524.777055:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204524.777059:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204524.777062:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204524.777066:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204524.777073:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204524.777075:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204524.777078:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204524.777086:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204524.777090:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5989) 00000100:00000001:2.0:1788204524.777161:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204524.777165:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204524.777167:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204524.777173:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777181:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204524.777182:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777185:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204524.777188:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204524.777190:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204524.777193:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205871232:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204524.777197:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204524.777200:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204524.777203:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777204:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204524.777208:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204524.777209:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204524.777214:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204524.777217:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204524.777221:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204524.777226:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205871232, portal 25 00000100:00000001:2.0:1788204524.777228:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204524.777229:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204524.777233:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777237:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204524.777240:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204524.777243:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204524.777246:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205871232, offset 0 00000400:00000200:2.0:1788204524.777252:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777260:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777265:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204524.777270:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204524.777274:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204524.777277:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204524.777280:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777283:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58662:58662:256:4294967295] 192.168.201.151@tcp LPNI seq info [58642:58642:8:4294967295] 00000400:00000200:2.0:1788204524.777287:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777296:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204524.777301:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777318:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204524.777325:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204524.777328:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204524.777334:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204524.777337:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204524.777340:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777343:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204524.777349:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204524.777353:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204524.777358:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204524.777360:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204524.777389:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204524.777394:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777402:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.777406:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777413:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.777414:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204524.777420:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204524.777424:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204524.777427:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300656130 00000100:00000010:2.0:1788204524.777432:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1960. 00000100:00000001:2.0:1788204524.777438:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204524.777443:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300656130 00000100:00000010:2.0:1788204524.777447:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c10c0. 00000100:00000001:2.0:1788204524.777450:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204524.777453:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204524.777456:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.777458:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204524.777463:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204524/real 1788204524] req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204524.777471:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204524.777473:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204524.777475:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204524.777477:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204524.777479:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204524.777482:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204524.777485:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204524.777494:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777500:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204524.777502:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777504:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.777506:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204524.777511:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777517:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204524.777522:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777528:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204524.777531:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777532:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.777533:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777535:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204524.777537:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204524.777540:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204524.777544:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204524.777546:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204524.777550:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204524.777558:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205871232:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204524.777563:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204524.777566:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777572:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204524.777573:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777575:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204524.777577:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204524.777578:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204524.777581:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871360:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204524.777584:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204524.777585:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204524.777587:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777589:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204524.777592:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204524.777594:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204524.777597:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204524.777600:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204524.777603:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204524.777605:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205871360, portal 10 00000100:00000001:2.0:1788204524.777607:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204524.777608:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204524.777615:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777621:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204524.777623:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204524.777625:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204524.777628:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205871360, offset 0 00000400:00000200:2.0:1788204524.777631:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777634:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777638:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204524.777641:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204524.777644:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204524.777646:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204524.777648:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777651:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58663:58663:256:4294967295] 192.168.201.151@tcp LPNI seq info [58643:58643:8:4294967295] 00000400:00000200:2.0:1788204524.777655:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777661:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204524.777665:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777668:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204524.777672:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204524.777674:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204524.777678:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204524.777681:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204524.777684:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777686:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204524.777689:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204524.777692:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204524.777695:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204524.777697:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204524.777700:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204524.777704:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777711:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.777714:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777720:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.777721:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204524.777725:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204524.777728:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204524.777731:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204524.777733:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300652187 00000100:00000001:2.0:1788204524.777736:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.777738:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300652187 00000100:00000001:2.0:1788204524.777740:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.777742:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204524.777744:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.777746:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204524.777762:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204524/real 1788204524] req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204524.777770:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204524.777772:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204524.777773:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204524.777776:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204524.777777:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204524.777780:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204524.777783:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204524.777790:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777803:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204524.777804:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777806:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.777807:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204524.777811:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777817:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204524.777821:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777827:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204524.777830:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777831:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.777832:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.777834:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204524.777836:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204524.777839:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204524.777842:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204524.777845:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204524.777849:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204524.777856:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871360:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204524.777860:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204524.777862:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777869:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204524.777870:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777872:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204524.777874:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204524.777876:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204524.777879:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871488:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204524.777882:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204524.777884:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204524.777885:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.777887:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204524.777890:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204524.777892:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204524.777895:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204524.777898:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204524.777901:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204524.777904:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205871488, portal 4 00000100:00000001:2.0:1788204524.777905:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204524.777907:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204524.777912:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.777918:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204524.777921:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204524.777923:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204524.777926:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205871488, offset 0 00000400:00000200:2.0:1788204524.777930:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777933:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777937:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204524.777939:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204524.777942:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204524.777944:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204524.777946:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204524.777949:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58664:58664:256:4294967295] 192.168.201.151@tcp LPNI seq info [58644:58644:8:4294967295] 00000400:00000200:2.0:1788204524.777953:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.777958:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204524.777963:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777967:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204524.777970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204524.777973:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204524.777976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204524.777978:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204524.777982:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.777984:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204524.777987:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204524.777990:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204524.777994:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204524.777996:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204524.777998:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204524.778002:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778010:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778014:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778021:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.778022:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204524.778026:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204524.778029:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204524.778032:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204524.778034:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300652073 00000100:00000001:2.0:1788204524.778036:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.778039:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300652073 00000100:00000001:2.0:1788204524.778040:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.778042:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204524.778044:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.778046:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204524.778049:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204524/real 1788204524] req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204524.778057:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204524.778059:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204524.778061:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204524.778063:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204524.778064:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204524.778066:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204524.778070:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204524.778079:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778085:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204524.778086:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778089:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.778090:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204524.778093:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778100:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204524.778105:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778111:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204524.778113:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778115:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.778116:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778118:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204524.778120:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204524.778122:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204524.778126:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204524.778128:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204524.778132:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204524.778140:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871488:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204524.778144:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204524.778147:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778153:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204524.778154:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.778156:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204524.778158:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204524.778160:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204524.778162:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204524.778165:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204524.778167:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204524.778169:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204524.778170:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204524.778174:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204524.778176:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204524.778178:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204524.778180:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204524.778183:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204524.778185:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205871616, portal 4 00000100:00000001:2.0:1788204524.778187:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204524.778188:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204524.778193:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778199:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204524.778201:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204524.778204:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204524.778206:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205871616, offset 0 00000400:00000200:2.0:1788204524.778210:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204524.778214:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.778217:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204524.778220:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204524.778222:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204524.778224:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204524.778226:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204524.778229:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58665:58665:256:4294967295] 192.168.201.151@tcp LPNI seq info [58645:58645:8:4294967295] 00000400:00000200:2.0:1788204524.778233:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204524.778239:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204524.778244:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.778247:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204524.778251:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204524.778254:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204524.778258:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204524.778260:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204524.778264:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204524.778266:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204524.778269:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204524.778272:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204524.778275:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204524.778276:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204524.778279:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204524.778282:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778289:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778292:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778297:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.778299:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204524.778301:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204524.778304:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204524.778307:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204524.778309:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300652079 00000100:00000001:2.0:1788204524.778312:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.778313:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300652079 00000100:00000001:2.0:1788204524.778315:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300656130 00000100:00000001:2.0:1788204524.778317:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204524.778319:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.778321:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204524.778324:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204524/real 1788204524] req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204524.778331:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204524.778333:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204524.778334:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204524.778337:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204524.778339:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204524.778341:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204524.778344:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204524.778352:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778358:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204524.778359:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778360:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.778362:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204524.778386:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778393:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204524.778397:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778403:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204524.778405:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778406:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204524.778407:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778409:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204524.778411:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204524.778413:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204524.778416:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204524.778418:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204524.778421:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204524.778428:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205871616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204524.778432:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778434:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778437:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205871232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778443:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204524.778445:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204524.778447:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204524.778450:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204524.778452:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204524.778453:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204524.778455:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204524.778458:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:2.0:1788204524.778462:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204524.778464:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204524.778466:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204524.778469:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204524.778470:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778472:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778475:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205871360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778487:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204524.778488:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204524.778489:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204524.778492:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204524.778493:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204524.778494:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204524.778496:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204524.778498:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:2.0:1788204524.778500:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204524.778501:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204524.778502:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204524.778504:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204524.778505:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778507:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778510:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205871488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778516:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204524.778517:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204524.778519:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204524.778521:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204524.778522:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204524.778523:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204524.778525:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204524.778526:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:2.0:1788204524.778529:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204524.778530:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204524.778531:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204524.778533:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204524.778534:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778536:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204524.778539:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205871616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204524.778544:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204524.778546:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204524.778547:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204524.778549:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204524.778550:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204524.778551:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204524.778553:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204524.778555:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:2.0:1788204524.778557:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204524.778558:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204524.778559:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204524.778561:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204524.778562:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778564:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204524.778571:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204524.778573:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204524.778574:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204524.778576:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204524.968487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204524.968494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204524.968499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204524.968501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204524.968504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204524.968505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204524.968508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204524.968509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204524.968526:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204524.968528:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204524.968531:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204525.288486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204525.288494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204525.288496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204525.288500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204525.288502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204525.288505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204525.288508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204525.288509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204525.288511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204525.736451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204525.736458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204525.736460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204525.736465:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204525.736468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204525.736472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204525.736474:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204525.736477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204525.736480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204525.800589:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204525.800592:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204525.800594:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204525.800596:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204525.800598:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204525.800601:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204525.800603:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204525.800604:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204525.800606:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204525.992466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204525.992471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204525.992475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204525.992476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204525.992478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204525.992479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204525.992482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204525.992483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204525.992548:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204525.992552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204525.992556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204526.312774:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204526.312781:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204526.312783:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.312787:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204526.312791:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204526.312794:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204526.312797:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204526.312800:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204526.312803:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204526.760493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204526.760500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204526.760503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204526.760509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204526.760513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204526.760516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204526.760519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204526.760521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204526.760524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204526.824476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204526.824484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204526.824487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204526.824494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204526.824501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204526.824507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204526.824510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204526.824514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204526.824518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204527.016507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204527.016514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204527.016519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204527.016521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204527.016524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204527.016525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204527.016528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204527.016529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204527.016547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204527.016549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204527.016552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204527.336438:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204527.336446:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204527.336448:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204527.336453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204527.336457:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204527.336462:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204527.336466:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204527.336468:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204527.336471:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204527.784426:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204527.784432:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204527.784434:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204527.784439:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204527.784443:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204527.784447:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204527.784449:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204527.784451:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204527.784454:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204527.848481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204527.848490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204527.848493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204527.848499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204527.848503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204527.848507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204527.848510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204527.848513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204527.848514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204528.040479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204528.040486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204528.040493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204528.040494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204528.040497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204528.040498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204528.040502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204528.040503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204528.040550:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204528.040554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204528.040556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204528.360491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204528.360499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204528.360501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204528.360507:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204528.360512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204528.360515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204528.360517:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204528.360520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204528.360523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204528.808493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204528.808499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204528.808501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204528.808506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204528.808511:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204528.808515:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204528.808518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204528.808520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204528.808523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204528.872437:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204528.872444:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204528.872446:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204528.872452:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204528.872456:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204528.872461:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204528.872465:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204528.872467:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204528.872470:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204529.064532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204529.064538:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204529.064540:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204529.064542:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064544:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204529.064546:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064547:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064549:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204529.064552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204529.064554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204529.064554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204529.064570:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204529.064573:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204529.064575:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204529.384458:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300656130, at 4300656640 00000100:00000001:2.0:1788204529.384459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204529.384463:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300656130, at 4300656640 00000100:00000001:2.0:1788204529.384464:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204529.384468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204529.384472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204529.384474:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00000001:2.0:1788204529.384474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788204529.384475:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00000001:2.0:1788204529.384478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204529.384481:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204529.384483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204529.384486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204529.832496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204529.832504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204529.832506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204529.832512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204529.832516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204529.832519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204529.832522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204529.832524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204529.832527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204529.896497:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204529.896505:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204529.896509:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204529.896513:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204529.896519:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204529.896528:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5984 00000400:00000200:2.0:1788204529.896538:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204529.896546:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204529.896550:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204529.896553:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896557:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.896560:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896563:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204529.896566:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204529.896568:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204529.896571:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204529.896575:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204529.896578:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204529.896580:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:0.0:1788204529.896583:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:2.0:1788204529.896584:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204529.896587:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204529.896588:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788204529.896589:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204529.896593:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000040:2.0:1788204529.896594:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204529.896596:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204529.896598:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788204529.896599:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204529.896603:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788204529.896604:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00000001:0.0:1788204529.896605:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204529.896610:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204529.896612:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896616:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204529.896619:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204529.896622:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204529.896625:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.896627:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204529.896629:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204529.896637:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204529.896640:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204529.896644:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204529.896645:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.896648:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204529.896650:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204529.896654:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204529.896673:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204529.896680:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204529.896688:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205871744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204529.896698:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204529.896703:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204529.896708:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205871744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204529.896727:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204529.896730:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204529.896733:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204529.896734:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204529.896736:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204529.896739:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204529.896742:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5984 00000400:00000200:2.0:1788204529.896747:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204529.896751:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204529.896754:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204529.896759:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896762:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.896768:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896771:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204529.896774:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204529.896777:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204529.896781:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204529.896784:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204529.896788:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204529.896794:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204529.896800:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896804:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.896809:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896814:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204529.896818:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204529.896825:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204529.896830:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204529.896834:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204529.896840:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204529.896844:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.896847:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204529.896852:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204529.896857:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.896861:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204529.896875:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204529.896879:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204529.896882:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204529.896885:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.896888:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204529.896892:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204529.896896:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204529.896901:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204529.896906:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204529.896915:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205871872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204529.896930:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204529.896934:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204529.896940:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205871872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204529.896955:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204529.896959:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204529.896965:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204529.896968:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204529.896970:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204529.896975:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204529.896978:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5984 00000400:00000200:2.0:1788204529.897019:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204529.897026:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204529.897031:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204529.897039:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897044:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.897048:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897052:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204529.897055:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204529.897058:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204529.897061:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204529.897065:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204529.897070:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.1:1788204529.897074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204529.897075:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.1:1788204529.897078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1788204529.897083:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897088:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.897094:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897098:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204529.897104:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204529.897109:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897116:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204529.897120:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204529.897125:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204529.897130:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897139:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204529.897143:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204529.897150:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897154:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204529.897160:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204529.897166:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204529.897172:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204529.897175:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897178:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204529.897181:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204529.897182:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204529.897188:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204529.897196:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204529.897206:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205872000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204529.897219:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204529.897224:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204529.897234:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205872000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204529.897258:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204529.897262:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204529.897266:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204529.897267:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204529.897269:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204529.897272:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204529.897278:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5984 00000400:00000200:2.0:1788204529.897288:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204529.897297:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204529.897302:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204529.897310:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897316:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.897322:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897328:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204529.897335:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204529.897339:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204529.897343:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204529.897348:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204529.897355:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204529.897361:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204529.897405:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897411:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204529.897417:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897421:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204529.897426:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204529.897435:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204529.897440:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204529.897444:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204529.897447:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204529.897454:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897459:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204529.897464:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204529.897471:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897476:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204529.897483:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204529.897489:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204529.897495:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204529.897499:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204529.897503:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204529.897506:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204529.897510:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204529.897520:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204529.897527:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204529.897536:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205872128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204529.897561:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204529.897566:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204529.897575:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205872128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204529.897597:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204529.897605:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5994) 00000800:00000200:3.1:1788204529.897630:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204529.897632:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204529.897883:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204529.897886:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204529.898124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204529.898125:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00010000:00000001:1.0:1788204530.088461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204530.088466:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204530.088470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204530.088471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204530.088473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204530.088474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204530.088476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204530.088477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204530.088538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204530.088541:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204530.088544:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204530.408469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204530.408476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204530.408478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204530.408484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204530.408487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204530.408489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204530.408491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204530.408493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204530.408495:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204530.856497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204530.856504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204530.856507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204530.856512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204530.856515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204530.856519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204530.856521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204530.856524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204530.856527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204530.920441:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204530.920448:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204530.920451:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204530.920456:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204530.920460:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204530.920464:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204530.920469:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204530.920475:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204530.920481:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204531.112523:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204531.112525:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204531.112529:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112533:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204531.112535:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112537:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112539:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204531.112544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204531.112548:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204531.112550:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204531.112551:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112553:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204531.112554:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112556:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112557:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204531.112562:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204531.112563:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204531.112565:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112566:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204531.112567:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112569:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204531.112570:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204531.112588:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204531.112591:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204531.112593:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204531.432508:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204531.432516:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204531.432519:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.432525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204531.432529:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204531.432533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204531.432536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204531.432539:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204531.432542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204531.880471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204531.880477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204531.880478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204531.880482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204531.880484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204531.880488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204531.880490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204531.880493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204531.880495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204531.944489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204531.944495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204531.944499:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204531.944504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204531.944508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204531.944511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204531.944517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204531.944524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204531.944526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204532.136489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204532.136495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204532.136503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204532.136504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204532.136509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204532.136510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204532.136516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204532.136518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204532.136574:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204532.136577:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204532.136579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204532.456481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204532.456487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204532.456489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204532.456493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204532.456496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204532.456501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204532.456503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204532.456506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204532.456509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204532.904466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204532.904473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204532.904475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204532.904481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204532.904485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204532.904489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204532.904491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204532.904494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204532.904502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204532.968494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204532.968499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204532.968502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204532.968507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204532.968511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204532.968515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204532.968519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204532.968522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204532.968525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204533.160517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204533.160521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204533.160524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204533.160525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204533.160527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204533.160528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204533.160530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204533.160531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204533.160543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204533.160545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204533.160546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204533.480467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204533.480473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204533.480474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204533.480477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204533.480480:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204533.480482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204533.480484:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204533.480486:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204533.480488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204533.928448:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204533.928455:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204533.928457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204533.928463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204533.928466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204533.928470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204533.928472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204533.928474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204533.928478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204533.992491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204533.992495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204533.992497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204533.992501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204533.992503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204533.992505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204533.992508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204533.992510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204533.992513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204534.184467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204534.184473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204534.184479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204534.184480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204534.184483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204534.184485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204534.184488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204534.184489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204534.184582:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204534.184613:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204534.184617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204534.504464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204534.504465:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300661249, at 4300661760 00000100:00000040:0.0:1788204534.504467:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300661249, at 4300661760 00000100:00000001:2.0:1788204534.504470:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204534.504474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204534.504478:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204534.504480:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204534.504481:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000010:0.0:1788204534.504483:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000001:2.0:1788204534.504483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204534.504489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204534.504491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204534.504493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204534.952453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204534.952471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204534.952473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204534.952478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204534.952482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204534.952485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204534.952488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204534.952490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204534.952493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204535.016458:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204535.016467:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204535.016471:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204535.016474:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204535.016475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204535.016479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204535.016480:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204535.016481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204535.016486:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5989 00000100:00100000:0.0:1788204535.016487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204535.016491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204535.016494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204535.016495:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204535.016502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1788204535.016503:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788204535.016504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204535.016510:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788204535.016511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.016517:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016521:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016524:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016527:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204535.016531:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204535.016533:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204535.016536:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204535.016543:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204535.016545:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204535.016547:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204535.016550:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016552:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016554:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016555:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204535.016558:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204535.016560:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016563:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204535.016565:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204535.016580:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204535.016583:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.016584:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204535.016586:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204535.016596:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204535.016599:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204535.016602:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204535.016603:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.016607:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204535.016609:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204535.016613:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204535.016621:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204535.016628:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204535.016637:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205872256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204535.016647:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204535.016651:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.016656:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205872256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204535.016675:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204535.016678:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204535.016680:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204535.016682:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204535.016683:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204535.016686:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204535.016688:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5989 00000400:00000200:2.0:1788204535.016693:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204535.016696:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204535.016699:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204535.016702:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016705:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016710:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016714:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204535.016717:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204535.016720:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204535.016724:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204535.016727:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204535.016731:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204535.016734:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204535.016739:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016742:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016745:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016747:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204535.016751:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204535.016756:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016761:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204535.016764:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204535.016770:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204535.016774:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.016778:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204535.016782:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204535.016785:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.016789:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204535.016796:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204535.016799:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204535.016802:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204535.016803:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.016806:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204535.016807:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204535.016809:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204535.016813:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204535.016824:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204535.016832:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205872384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204535.016844:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204535.016847:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.016851:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205872384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204535.016861:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204535.016863:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204535.016865:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204535.016868:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204535.016870:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204535.016875:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204535.016877:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5989 00000400:00000200:2.0:1788204535.016917:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204535.016925:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204535.016929:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204535.016935:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016939:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016944:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016947:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204535.016951:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204535.016952:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204535.016954:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204535.016958:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204535.016963:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204535.016968:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204535.016973:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204535.016974:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204535.016976:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.016979:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204535.016981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1788204535.016982:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204535.016986:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204535.016991:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204535.016994:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204535.016997:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204535.016999:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204535.017002:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017003:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204535.017005:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204535.017011:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017013:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204535.017017:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204535.017021:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204535.017023:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204535.017024:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017028:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204535.017029:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204535.017033:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204535.017038:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204535.017044:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204535.017054:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205872512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204535.017068:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204535.017072:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.017078:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205872512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204535.017088:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204535.017090:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204535.017092:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204535.017095:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204535.017096:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204535.017100:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204535.017104:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5989 00000400:00000200:2.0:1788204535.017109:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204535.017112:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204535.017114:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204535.017118:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.017120:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.017121:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.017122:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204535.017124:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204535.017125:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204535.017128:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204535.017131:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204535.017132:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204535.017134:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204535.017136:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.017141:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204535.017142:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204535.017144:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204535.017147:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204535.017150:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204535.017154:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204535.017155:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204535.017164:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204535.017168:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017172:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204535.017174:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204535.017178:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017182:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204535.017187:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204535.017192:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204535.017197:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204535.017201:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.017206:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204535.017209:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204535.017211:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204535.017215:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204535.017219:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204535.017225:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205872640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204535.017236:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204535.017239:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.017245:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205872640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204535.017252:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204535.017258:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5999) 00000800:00000200:3.1:1788204535.017504:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204535.017505:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204535.017760:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204535.017762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204535.017965:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204535.017968:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00010000:00000001:1.0:1788204535.208507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204535.208514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204535.208519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204535.208520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204535.208522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204535.208523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204535.208525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204535.208526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204535.208540:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204535.208543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204535.208545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204535.976464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204535.976470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204535.976471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204535.976475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204535.976477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204535.976480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204535.976482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204535.976484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204535.976485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.040458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204536.040466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204536.040469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.040478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204536.040485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204536.040492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204536.040507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204536.040511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204536.040516:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204536.232490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204536.232495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204536.232500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204536.232501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204536.232503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204536.232504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204536.232506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204536.232507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204536.232580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204536.232587:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204536.232591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204537.000487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204537.000494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204537.000496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204537.000501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204537.000504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204537.000509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204537.000511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204537.000514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204537.000516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204537.064473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204537.064480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204537.064484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204537.064490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204537.064496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204537.064501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204537.064513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204537.064517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204537.064520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204537.256597:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204537.256607:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204537.256615:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204537.256617:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204537.256620:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204537.256622:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204537.256625:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204537.256627:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204537.256657:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204537.256660:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204537.256664:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204538.024460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204538.024470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204538.024472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204538.024479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204538.024483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204538.024488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204538.024491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204538.024495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204538.024499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204538.088446:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204538.088452:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204538.088455:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204538.088462:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204538.088469:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204538.088476:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204538.088482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204538.088487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204538.088491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204538.280487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204538.280496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204538.280501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204538.280503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204538.280506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204538.280507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204538.280510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204538.280511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204538.280595:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204538.280600:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204538.280604:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204539.048554:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204539.048565:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204539.048568:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204539.048575:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204539.048580:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204539.048585:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204539.048589:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204539.048592:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204539.048596:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204539.112494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204539.112509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204539.112511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204539.112513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204539.112516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204539.112519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204539.112522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204539.112527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204539.112532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204539.304540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304544:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204539.304547:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204539.304550:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204539.304553:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304555:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204539.304557:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304558:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304560:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304562:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204539.304563:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204539.304566:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204539.304566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204539.304582:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204539.304584:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204539.304585:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204539.624456:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300666369, at 4300666880 00000100:00000040:1.0:1788204539.624456:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300666369, at 4300666880 00000100:00000010:1.0:1788204539.624473:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000010:0.0:1788204539.624476:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000001:1.0:1788204540.072519:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204540.072526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204540.072528:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204540.072533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204540.072536:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204540.072540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204540.072543:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204540.072545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204540.072547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204540.136489:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204540.136501:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204540.136503:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204540.136506:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204540.136511:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204540.136517:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 5994 00000400:00000200:0.0:1788204540.136537:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204540.136544:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204540.136548:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204540.136556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136559:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.136565:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136571:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204540.136584:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204540.136591:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204540.136596:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204540.136600:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204540.136603:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204540.136607:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204540.136611:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136614:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.136616:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136618:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204540.136621:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204540.136624:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136625:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.136628:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204540.136630:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204540.136633:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136635:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204540.136637:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204540.136653:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204540.136658:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204540.136663:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204540.136665:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136673:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204540.136676:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204540.136681:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204540.136691:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204540.136698:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204540.136712:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204540.136723:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204540.136728:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204540.136733:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204540.136764:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204540.136767:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204540.136770:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204540.136771:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204540.136773:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204540.136776:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204540.136779:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5994 00000400:00000200:0.0:1788204540.136785:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204540.136792:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204540.136795:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204540.136795:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788204540.136799:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204540.136801:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204540.136802:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.136805:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136807:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204540.136808:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204540.136809:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204540.136810:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204540.136810:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1788204540.136812:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204540.136814:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204540.136817:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204540.136819:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204540.136820:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136824:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.136828:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.136831:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204540.136836:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204540.136838:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204540.136839:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204540.136841:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136842:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.136845:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204540.136846:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204540.136849:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204540.136851:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204540.136852:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.136857:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136859:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:2.0:1788204540.136861:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205872768:192.168.201.151@tcp:250:kworker.0 00000008:00100000:0.0:1788204540.136864:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204540.136868:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.136868:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204540.136872:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204540.136875:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 02000000:00000001:2.0:1788204540.136876:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1788204540.136877:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204540.136881:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204540.136881:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.136883:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136886:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204540.136887:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204540.136887:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204540.136891:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204540.136896:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0ec00. 02000000:00000010:0.0:1788204540.136899:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788204540.136900:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.136903:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204540.136911:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204540.136911:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204540.136920:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00000001:0.0:1788204540.136922:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204540.136926:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204540.136930:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000040:0.0:1788204540.136931:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204540.136941:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205872768, portal 25 00000100:00000001:0.0:1788204540.136943:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.136947:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788204540.136949:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204540.136951:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00080000:0.0:1788204540.136955:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204540.136958:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204540.136961:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204540.136964:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:2.0:1788204540.136967:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.136969:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5994 00000400:00000200:0.0:1788204540.136978:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204540.136985:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204540.136985:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788204540.136990:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204540.136993:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204540.136997:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000040:0.0:1788204540.136998:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:2.0:1788204540.137001:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205872768, offset 0 00000100:00000001:0.0:1788204540.137007:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204540.137010:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204540.137011:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137015:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204540.137018:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204540.137019:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204540.137020:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204540.137022:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:2.0:1788204540.137024:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788204540.137025:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204540.137029:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204540.137032:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137033:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.137037:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137037:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204540.137038:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204540.137042:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204540.137045:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137047:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.137049:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204540.137049:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1788204540.137053:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204540.137054:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.137056:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204540.137058:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204540.137059:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.137062:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204540.137064:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204540.137066:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204540.137070:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1788204540.137070:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:0.0:1788204540.137072:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.137074:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204540.137075:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204540.137077:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204540.137078:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000010:0.0:1788204540.137081:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000400:00000200:2.0:1788204540.137085:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137086:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204540.137092:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204540.137096:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58674:58674:256:4294967295] 192.168.201.151@tcp LPNI seq info [58654:58654:8:4294967295] 00000100:00000001:0.0:1788204540.137106:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204540.137108:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204540.137111:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204540.137115:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204540.137123:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204540.137128:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204540.137130:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:0.0:1788204540.137137:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204540.137140:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.0:1788204540.137141:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204540.137143:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204540.137148:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:2.0:1788204540.137148:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000100:00080000:0.0:1788204540.137154:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 5994 00000800:00000200:2.0:1788204540.137159:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000400:00000200:0.0:1788204540.137162:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:2.0:1788204540.137165:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000400:00000200:0.0:1788204540.137168:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204540.137172:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1788204540.137175:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204540.137177:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204540.137179:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1788204540.137180:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.137185:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137187:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000100:2.0:1788204540.137189:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788204540.137190:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204540.137192:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000010:2.0:1788204540.137193:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000020:00000001:0.0:1788204540.137194:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204540.137197:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204540.137199:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204540.137201:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204540.137201:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1788204540.137207:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137211:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788204540.137212:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204540.137217:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204540.137220:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204540.137220:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000020:00000040:0.0:1788204540.137223:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204540.137226:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:2.0:1788204540.137226:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:0.0:1788204540.137230:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204540.137232:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204540.137234:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204540.137235:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1788204540.137236:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.137240:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204540.137241:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204540.137244:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204540.137245:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204540.137246:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204540.137250:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204540.137260:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204540.137262:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788204540.137264:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204540.137266:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204540.137268:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204540.137269:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204540.137270:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204540.137272:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204540.137277:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204540.137283:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137283:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137287:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1788204540.137290:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204540.137295:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204540.137296:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204540.137299:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204540.137302:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204540.137305:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137308:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300671489 00000100:00000010:2.0:1788204540.137315:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1de0. 00000100:00000001:0.0:1788204540.137317:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137324:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788204540.137325:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6005) 00000100:00000001:2.0:1788204540.137333:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300671489 00000100:00000010:2.0:1788204540.137338:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1000. 00000100:00000001:2.0:1788204540.137342:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204540.137346:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204540.137349:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137352:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204540.137359:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204540.137389:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204540.137391:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204540.137392:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204540.137394:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204540.137396:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204540.137398:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204540.137401:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204540.137407:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137411:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204540.137413:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137415:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.137417:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204540.137419:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137423:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204540.137427:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137430:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204540.137432:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137434:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.137434:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137436:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204540.137438:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204540.137440:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204540.137443:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204540.137444:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204540.137447:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204540.137452:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205872768:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204540.137455:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137457:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.137459:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205872768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137463:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204540.137465:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204540.137467:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204540.137469:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204540.137470:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204540.137472:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204540.137473:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204540.137476:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:2.0:1788204540.137493:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204540.137495:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204540.137497:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204540.137500:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204540.137502:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137504:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137512:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204540.137514:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137517:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204540.137518:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204540.137519:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204540.137523:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137530:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204540.137531:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.137533:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204540.137535:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204540.137538:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204540.137541:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205872896:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204540.137544:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204540.137546:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204540.137548:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.137549:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204540.137552:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204540.137554:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204540.137557:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204540.137558:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204540.137561:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204540.137563:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205872896, portal 10 00000100:00000001:2.0:1788204540.137565:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204540.137566:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204540.137573:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137580:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204540.137582:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204540.137584:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204540.137586:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205872896, offset 0 00000400:00000200:2.0:1788204540.137590:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137594:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.137598:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204540.137601:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204540.137604:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204540.137606:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204540.137608:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137611:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58675:58675:256:4294967295] 192.168.201.151@tcp LPNI seq info [58655:58655:8:4294967295] 00000400:00000200:2.0:1788204540.137615:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.137621:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204540.137626:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.137629:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204540.137633:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204540.137635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204540.137638:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204540.137654:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204540.137657:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.137658:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204540.137660:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204540.137663:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204540.137665:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204540.137666:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204540.137669:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204540.137671:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137677:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.137678:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137686:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137688:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204540.137691:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204540.137694:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204540.137697:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204540.137699:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300667547 00000100:00000001:2.0:1788204540.137701:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.137703:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300667547 00000100:00000001:2.0:1788204540.137705:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.137707:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204540.137709:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137711:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204540.137714:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204540.137722:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204540.137723:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204540.137724:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204540.137726:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204540.137727:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204540.137728:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204540.137730:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204540.137735:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137739:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204540.137739:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137740:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.137741:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204540.137743:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137748:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204540.137750:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137754:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204540.137756:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137756:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.137757:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.137758:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204540.137760:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204540.137761:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204540.137763:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204540.137764:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204540.137766:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204540.137770:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205872896:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204540.137773:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204540.137775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137779:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204540.137780:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.137781:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204540.137782:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204540.137783:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204540.137785:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205873024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204540.137786:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204540.137787:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204540.137789:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.137790:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204540.137792:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204540.137793:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204540.137796:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204540.137797:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204540.137800:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204540.137801:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205873024, portal 4 00000100:00000001:2.0:1788204540.137803:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204540.137804:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204540.137810:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137816:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204540.137819:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204540.137821:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204540.137824:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205873024, offset 0 00000400:00000200:2.0:1788204540.137827:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137831:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.137835:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204540.137837:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204540.137840:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204540.137842:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204540.137844:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204540.137847:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58676:58676:256:4294967295] 192.168.201.151@tcp LPNI seq info [58656:58656:8:4294967295] 00000400:00000200:2.0:1788204540.137851:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.137857:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204540.137862:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.137891:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204540.137897:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204540.137899:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204540.137903:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204540.137906:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204540.137910:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.137912:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204540.137915:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204540.137918:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204540.137922:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204540.137923:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204540.137926:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204540.137930:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137937:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.137940:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.137947:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137948:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204540.137951:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204540.137955:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204540.137958:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204540.137960:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300667433 00000100:00000001:2.0:1788204540.137962:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.137964:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300667433 00000100:00000001:2.0:1788204540.137966:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.137969:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204540.137971:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.137973:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204540.137976:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204540.137985:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204540.137986:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204540.137988:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204540.137990:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204540.137992:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204540.137995:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204540.137998:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204540.138005:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138012:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204540.138013:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138015:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.138017:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204540.138020:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138027:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204540.138030:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138036:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204540.138039:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138040:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.138042:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138043:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204540.138046:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204540.138048:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204540.138052:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204540.138054:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204540.138058:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204540.138066:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205873024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204540.138069:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204540.138072:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138079:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204540.138081:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.138083:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204540.138085:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204540.138087:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204540.138090:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205873152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204540.138092:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204540.138094:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204540.138096:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204540.138098:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204540.138101:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204540.138103:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204540.138106:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204540.138109:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204540.138113:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204540.138115:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205873152, portal 4 00000100:00000001:2.0:1788204540.138117:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204540.138118:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204540.138124:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138130:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204540.138132:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204540.138134:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204540.138152:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205873152, offset 0 00000400:00000200:2.0:1788204540.138157:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204540.138162:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.138165:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204540.138188:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204540.138191:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204540.138193:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204540.138196:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204540.138199:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58677:58677:256:4294967295] 192.168.201.151@tcp LPNI seq info [58657:58657:8:4294967295] 00000400:00000200:2.0:1788204540.138203:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204540.138209:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204540.138213:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.138215:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204540.138219:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204540.138221:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204540.138225:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204540.138228:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204540.138231:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204540.138233:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204540.138236:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204540.138239:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204540.138242:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204540.138244:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204540.138247:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204540.138250:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138258:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.138261:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204565 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138268:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.138269:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204540.138272:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204540.138274:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204540.138276:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204540.138277:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300667439 00000100:00000001:2.0:1788204540.138279:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.138281:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300667439 00000100:00000001:2.0:1788204540.138282:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300671489 00000100:00000001:2.0:1788204540.138283:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204540.138285:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.138286:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204540.138288:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204540.138294:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204540.138294:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204540.138296:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204540.138297:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204540.138298:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204540.138300:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204540.138302:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204540.138306:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138311:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204540.138311:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138312:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.138313:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204540.138315:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138318:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204540.138320:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138324:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204540.138325:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138326:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204540.138326:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138328:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204540.138329:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204540.138331:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204540.138335:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204540.138337:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204540.138341:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204540.138349:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205873152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204540.138352:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138354:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.138357:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205872896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138385:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204540.138386:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204540.138387:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204540.138389:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204540.138390:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204540.138391:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204540.138393:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204540.138394:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788204540.138397:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204540.138398:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204540.138399:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204540.138401:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204540.138402:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138403:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.138406:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205873024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138410:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204540.138411:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204540.138412:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204540.138415:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204540.138416:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204540.138418:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204540.138420:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204540.138421:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:2.0:1788204540.138423:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204540.138424:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204540.138426:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204540.138428:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204540.138429:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138430:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204540.138432:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205873152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204540.138435:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204540.138436:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204540.138437:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204540.138438:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204540.138438:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204540.138439:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204540.138440:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204540.138441:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:2.0:1788204540.138442:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204540.138443:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204540.138443:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204540.138445:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204540.138446:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138447:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.138450:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204540.138450:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.138452:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204540.138453:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204540.328466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204540.328476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204540.328507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204540.328508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204540.328511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204540.328512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204540.328515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204540.328516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204540.328573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204540.328578:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204540.328580:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204541.096517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204541.096525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204541.096526:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204541.096531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204541.096534:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204541.096538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204541.096540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204541.096542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204541.096544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204541.160521:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204541.160530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204541.160532:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204541.160541:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204541.160545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204541.160550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204541.160554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204541.160557:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204541.160559:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204541.352542:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204541.352545:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204541.352548:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352553:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204541.352555:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352557:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352558:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204541.352562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204541.352565:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204541.352567:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204541.352568:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352570:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204541.352571:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352573:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352575:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352578:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204541.352580:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204541.352581:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204541.352583:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352584:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204541.352585:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352587:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204541.352589:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204541.352610:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204541.352613:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204541.352616:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204542.120520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204542.120531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204542.120534:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204542.120541:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204542.120545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204542.120550:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204542.120554:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204542.120557:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204542.120560:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204542.184502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204542.184511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204542.184515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204542.184522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204542.184528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204542.184535:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204542.184542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204542.184546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204542.184550:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204542.376463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204542.376470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204542.376475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204542.376476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204542.376478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204542.376479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204542.376482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204542.376484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204542.376538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204542.376543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204542.376546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204543.144493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204543.144501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204543.144503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204543.144509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204543.144512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204543.144516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204543.144518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204543.144520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204543.144522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204543.208533:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204543.208543:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204543.208547:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204543.208556:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204543.208562:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204543.208570:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204543.208586:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204543.208590:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204543.208595:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204543.400539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204543.400545:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204543.400549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204543.400550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204543.400553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204543.400554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204543.400556:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204543.400557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204543.400573:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204543.400575:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204543.400577:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204544.168523:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204544.168532:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204544.168533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204544.168539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204544.168542:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204544.168546:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204544.168548:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204544.168550:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204544.168553:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.232462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204544.232472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204544.232475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204544.232482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204544.232488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204544.232494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204544.232499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204544.232504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204544.232508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204544.424514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204544.424524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204544.424532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204544.424533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204544.424537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204544.424538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204544.424542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204544.424543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204544.424608:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204544.424613:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204544.424616:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204544.744447:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300671489, at 4300672000 00000100:00000040:0.0:1788204544.744454:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300671489, at 4300672000 00000100:00000010:1.0:1788204544.744463:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000010:0.0:1788204544.744466:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000001:1.0:1788204545.192456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204545.192461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204545.192463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204545.192468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204545.192470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204545.192473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204545.192475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204545.192477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204545.192479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204545.256537:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204545.256547:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204545.256552:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204545.256559:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204545.256565:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204545.256567:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204545.256575:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6000 00000100:00000001:0.0:1788204545.256584:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204545.256587:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204545.256597:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204545.256604:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1788204545.256605:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204545.256609:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204545.256615:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788204545.256616:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204545.256622:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204545.256623:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788204545.256631:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.256633:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256638:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.256642:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256647:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204545.256654:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204545.256656:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204545.256660:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204545.256663:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204545.256666:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204545.256668:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204545.256672:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256674:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.256677:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256679:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204545.256682:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204545.256686:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256688:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204545.256691:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204545.256695:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204545.256698:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.256701:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204545.256704:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204545.256720:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204545.256724:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204545.256728:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204545.256729:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.256733:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204545.256736:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204545.256740:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204545.256747:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204545.256754:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204545.256764:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205873280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204545.256780:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204545.256784:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.256794:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205873280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204545.256817:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204545.256820:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204545.256824:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204545.256825:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204545.256833:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204545.256837:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204545.256840:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6000 00000400:00000200:2.0:1788204545.256845:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204545.256849:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204545.256852:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204545.256858:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256862:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.256869:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256871:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204545.256877:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204545.256880:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204545.256884:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204545.256888:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204545.256893:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204545.256897:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204545.256903:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256908:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.256913:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256918:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204545.256925:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204545.256931:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204545.256936:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204545.256940:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204545.256948:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204545.256953:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.256958:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204545.256972:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204545.256977:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.256981:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204545.256986:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204545.256991:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204545.256997:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204545.257000:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257003:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204545.257007:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204545.257010:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204545.257016:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204545.257022:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204545.257032:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205873408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204545.257045:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204545.257049:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.257056:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205873408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204545.257071:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204545.257076:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204545.257082:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204545.257086:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204545.257090:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204545.257096:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204545.257102:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6000 00000400:00000200:2.0:1788204545.257114:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204545.257122:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204545.257127:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204545.257142:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257146:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.257149:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257154:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204545.257159:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204545.257161:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204545.257165:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204545.257168:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000200:3.1:1788204545.257170:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204545.257171:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204545.257174:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204545.257178:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257179:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.257181:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257183:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204545.257187:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.1:1788204545.257190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00080000:2.0:1788204545.257192:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257197:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204545.257200:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204545.257204:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204545.257208:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257212:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204545.257215:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204545.257221:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257224:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204545.257229:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204545.257233:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204545.257236:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204545.257240:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257244:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204545.257246:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204545.257250:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204545.257256:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788204545.257262:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204545.257271:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205873536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204545.257294:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204545.257299:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.257306:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205873536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204545.257331:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204545.257335:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204545.257341:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204545.257346:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204545.257348:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204545.257353:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204545.257358:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6000 00000400:00000200:2.0:1788204545.257413:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204545.257422:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204545.257426:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204545.257433:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257438:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.257443:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257447:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204545.257452:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204545.257454:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204545.257458:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204545.257462:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204545.257467:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204545.257473:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204545.257481:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257488:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204545.257492:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257497:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204545.257500:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204545.257503:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204545.257508:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204545.257513:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204545.257517:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204545.257523:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257527:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204545.257531:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204545.257535:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257540:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204545.257546:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204545.257552:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204545.257559:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204545.257562:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.257567:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204545.257570:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204545.257574:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204545.257581:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204545.257587:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204545.257599:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205873664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204545.257616:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204545.257620:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.257627:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205873664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204545.257641:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204545.257648:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6010) 00000800:00000200:3.1:1788204545.257825:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204545.257828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204545.258110:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204545.258112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204545.258413:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204545.258415:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00010000:00000001:1.0:1788204545.448523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204545.448529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204545.448532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204545.448533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204545.448535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204545.448536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204545.448538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204545.448539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204545.448553:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204545.448555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204545.448557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204546.216515:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204546.216526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204546.216529:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204546.216535:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204546.216539:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204546.216544:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204546.216546:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204546.216550:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204546.216553:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204546.280479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204546.280485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204546.280488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204546.280495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204546.280501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204546.280506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204546.280518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204546.280521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204546.280523:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204546.472525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204546.472533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204546.472537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204546.472538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204546.472541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204546.472541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204546.472546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204546.472548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204546.472610:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204546.472613:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204546.472616:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204547.240517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204547.240523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204547.240525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204547.240530:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204547.240533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204547.240537:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204547.240539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204547.240542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204547.240544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204547.304515:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204547.304526:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204547.304529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204547.304536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204547.304540:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204547.304541:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204547.304550:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204547.304551:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204547.304555:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204547.304557:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204547.304561:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204547.304564:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204547.304566:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204547.304576:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204547.304580:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204547.304584:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204547.304587:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204547.304591:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204547.496524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204547.496534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204547.496542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204547.496543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204547.496547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204547.496548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204547.496552:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204547.496553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204547.496572:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204547.496574:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204547.496577:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204547.816516:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204547.816525:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204547.816527:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204547.816533:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204547.816536:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204547.816540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204547.816542:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204547.816545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204547.816547:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204548.264479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204548.264485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204548.264486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204548.264490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204548.264492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204548.264495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204548.264496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204548.264498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204548.264500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204548.328433:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204548.328440:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204548.328442:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204548.328447:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204548.328450:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204548.328453:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204548.328455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204548.328456:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204548.328457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204548.328458:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204548.328459:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204548.328461:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204548.328461:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204548.328462:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204548.328464:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204548.328466:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204548.328467:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204548.328469:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204548.520487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204548.520493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204548.520499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204548.520500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204548.520502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204548.520503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204548.520506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204548.520508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204548.520571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204548.520575:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204548.520579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204548.840439:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204548.840446:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204548.840448:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204548.840453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204548.840464:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204548.840468:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204548.840475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204548.840477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204548.840479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204549.288488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204549.288493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204549.288495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204549.288498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204549.288501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204549.288504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204549.288506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204549.288509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204549.288512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204549.352473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204549.352473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204549.352480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204549.352481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204549.352483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204549.352485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204549.352489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204549.352491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204549.352494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204549.352496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204549.352498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204549.352500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204549.352501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204549.352506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204549.352509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204549.352511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204549.352514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204549.352517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204549.544509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204549.544518:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204549.544521:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204549.544524:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544528:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204549.544530:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544533:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544535:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204549.544539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204549.544543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204549.544544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204549.544563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204549.544566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204549.544569:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204549.864456:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300676609, at 4300677120 00000100:00000001:2.0:1788204549.864456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204549.864462:0:48409:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300676609, at 4300677120 00000100:00000001:2.0:1788204549.864463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204549.864467:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204549.864469:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000010:0.0:1788204549.864472:0:48409:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00100000:2.0:1788204549.864473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204549.864478:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204549.864482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204549.864485:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204549.864488:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204549.864492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204550.312472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204550.312478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204550.312480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204550.312483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204550.312485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204550.312488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204550.312490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204550.312491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204550.312493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204550.376474:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204550.376474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204550.376474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204550.376480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204550.376481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204550.376484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204550.376484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204550.376491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204550.376491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204550.376493:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204550.376495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204550.376497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204550.376498:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204550.376499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204550.376501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204550.376503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204550.376504:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204550.376506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204550.376510:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204550.376510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204550.376521:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6005 00000400:00000200:0.0:1788204550.376534:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204550.376535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204550.376539:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204550.376544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204550.376546:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204550.376552:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204550.376556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376561:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.376565:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376568:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204550.376571:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204550.376573:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204550.376576:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204550.376579:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204550.376580:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204550.376582:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204550.376585:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376586:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.376587:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376589:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204550.376592:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204550.376596:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376598:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204550.376601:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204550.376605:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204550.376609:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.376611:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204550.376614:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204550.376625:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204550.376638:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204550.376643:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204550.376645:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.376648:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204550.376651:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204550.376656:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204550.376665:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204550.376672:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204550.376682:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205873792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204550.376695:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204550.376698:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204550.376701:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205873792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204550.376717:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204550.376719:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204550.376722:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204550.376724:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204550.376725:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204550.376728:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204550.376731:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6005 00000400:00000200:0.0:1788204550.376735:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204550.376739:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204550.376741:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204550.376744:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376746:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.376751:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376755:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204550.376760:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204550.376762:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204550.376765:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204550.376770:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204550.376774:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204550.376777:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204550.376783:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376789:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.376793:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376797:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204550.376802:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204550.376810:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376815:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204550.376818:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204550.376824:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204550.376828:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.376831:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204550.376835:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204550.376837:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.376841:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204550.376847:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204550.376850:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204550.376852:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204550.376854:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.376856:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204550.376857:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204550.376861:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204550.376867:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204550.376872:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204550.376884:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205873920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204550.376907:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204550.376912:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204550.376920:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205873920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204550.376937:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204550.376941:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204550.376947:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204550.376950:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204550.376952:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204550.376955:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204550.376960:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6005 00000400:00000200:0.0:1788204550.376967:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204550.376972:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204550.376975:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204550.376977:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376981:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.376984:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.376986:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204550.376988:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204550.376990:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204550.376993:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204550.376996:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204550.377000:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204550.377004:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204550.377011:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377014:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.377017:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377022:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204550.377033:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204550.377041:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.1:1788204550.377044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204550.377046:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204550.377049:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 02000000:00000001:0.0:1788204550.377050:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204550.377055:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204550.377059:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377063:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204550.377067:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204550.377070:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377074:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204550.377081:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204550.377086:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204550.377091:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204550.377094:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377098:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204550.377101:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204550.377104:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204550.377109:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204550.377112:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204550.377120:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205874048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204550.377136:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204550.377140:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204550.377144:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205874048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204550.377152:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204550.377154:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204550.377156:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204550.377160:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204550.377161:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204550.377163:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204550.377165:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6005 00000400:00000200:0.0:1788204550.377170:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204550.377174:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204550.377176:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204550.377179:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377182:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.377184:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377188:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204550.377190:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204550.377192:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204550.377194:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204550.377195:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204550.377198:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204550.377200:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204550.377204:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377207:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204550.377210:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377213:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204550.377217:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204550.377221:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204550.377223:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204550.377225:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204550.377229:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204550.377232:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377236:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204550.377238:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204550.377242:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377243:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204550.377249:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204550.377253:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204550.377255:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204550.377257:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204550.377260:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204550.377261:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204550.377264:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204550.377268:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204550.377271:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204550.377277:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205874176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204550.377287:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204550.377291:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204550.377298:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205874176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204550.377334:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204550.377342:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6015) 00000800:00000200:3.1:1788204550.377564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204550.377565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204550.377766:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204550.377768:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204550.377928:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204550.377931:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204550.568459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204550.568467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204550.568472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204550.568473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204550.568477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204550.568478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204550.568481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204550.568482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204550.568542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204550.568547:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204550.568550:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204550.888454:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204550.888460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204550.888461:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204550.888465:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204550.888468:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204550.888471:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204550.888473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204550.888475:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204550.888478:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204551.336491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204551.336499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204551.336501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204551.336506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204551.336509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204551.336513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204551.336515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204551.336517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204551.336520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204551.400475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204551.400481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204551.400483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204551.400485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204551.400487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204551.400489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204551.400496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204551.400496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204551.400502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204551.400503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204551.400507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204551.400508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204551.400514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204551.400517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204551.400520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204551.400521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204551.400525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204551.400528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204551.592490:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204551.592492:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204551.592495:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592499:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204551.592500:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592502:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592503:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204551.592507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204551.592510:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204551.592511:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204551.592512:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592513:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204551.592514:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592515:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592516:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204551.592519:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204551.592520:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204551.592521:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592522:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204551.592523:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592524:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204551.592525:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204551.592538:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204551.592541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204551.592543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204551.912487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204551.912495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204551.912497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204551.912503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204551.912506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204551.912510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204551.912513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204551.912515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204551.912518:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204552.360500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204552.360508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204552.360510:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204552.360516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204552.360519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204552.360523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204552.360526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204552.360529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204552.360531:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204552.424492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204552.424494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204552.424501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204552.424502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204552.424504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.424506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204552.424511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204552.424512:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204552.424516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204552.424518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204552.424522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204552.424523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204552.424528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204552.424532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204552.424535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.424537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204552.424542:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204552.424545:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204552.616490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204552.616496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204552.616502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204552.616504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204552.616507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204552.616509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204552.616512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204552.616513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204552.616577:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204552.616581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204552.616585:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204552.936486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204552.936493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204552.936495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204552.936501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204552.936504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204552.936508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204552.936511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204552.936513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204552.936516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204553.384474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204553.384482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204553.384484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204553.384490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204553.384493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204553.384497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204553.384499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204553.384502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204553.384505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204553.448460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204553.448460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204553.448466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204553.448467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204553.448469:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204553.448470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204553.448475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204553.448476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204553.448480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204553.448481:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204553.448483:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204553.448485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204553.448487:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204553.448489:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204553.448491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204553.448496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204553.448499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204553.448501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204553.640522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204553.640528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204553.640534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204553.640535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204553.640538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204553.640539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204553.640542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204553.640543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204553.640562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204553.640564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204553.640566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204553.960465:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204553.960473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204553.960476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204553.960481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204553.960485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204553.960489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204553.960492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204553.960494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204553.960497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204554.408485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204554.408494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204554.408496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204554.408502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204554.408505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204554.408509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204554.408512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204554.408514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204554.408517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204554.472461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204554.472461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204554.472468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204554.472468:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204554.472470:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204554.472472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204554.472474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204554.472477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204554.472478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204554.472482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204554.472483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204554.472484:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204554.472487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204554.472488:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204554.472494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204554.472494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204554.472498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204554.472501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204554.664478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204554.664484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204554.664488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204554.664489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204554.664491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204554.664493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204554.664496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204554.664497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204554.664553:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204554.664557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204554.664560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204554.984479:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300681729, at 4300682240 00000100:00000040:1.0:1788204554.984479:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300681729, at 4300682240 00000100:00000001:2.0:1788204554.984479:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204554.984486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204554.984490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204554.984493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204554.984496:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000001:2.0:1788204554.984497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788204554.984500:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000001:2.0:1788204554.984503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204554.984506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204554.984509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204554.984512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.432457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204555.432464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204555.432467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204555.432471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204555.432474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204555.432478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204555.432485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204555.432487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204555.432490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204555.496457:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204555.496463:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204555.496465:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204555.496466:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788204555.496471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:1.0:1788204555.496474:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204555.496476:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204555.496477:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204555.496479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204555.496482:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204555.496482:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204555.496485:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204555.496490:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204555.496490:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204555.496500:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6010 00000400:00000200:1.0:1788204555.496512:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204555.496521:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204555.496526:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204555.496530:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496533:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.496536:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496538:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204555.496542:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204555.496544:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204555.496547:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204555.496550:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204555.496552:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204555.496555:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204555.496558:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496560:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.496562:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496564:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204555.496567:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204555.496571:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496573:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204555.496576:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204555.496579:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204555.496582:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.496585:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204555.496587:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204555.496595:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788204555.496599:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204555.496602:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204555.496604:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.496608:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204555.496610:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204555.496613:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204555.496620:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788204555.496626:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204555.496635:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205874304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204555.496645:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204555.496649:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204555.496654:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205874304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204555.496675:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204555.496677:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204555.496681:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204555.496682:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204555.496684:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204555.496687:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204555.496689:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6010 00000400:00000200:1.0:1788204555.496694:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204555.496698:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204555.496701:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204555.496707:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496712:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.496717:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496723:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204555.496728:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204555.496732:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204555.496737:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204555.496741:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204555.496745:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204555.496751:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204555.496761:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496766:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.496771:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496777:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204555.496782:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204555.496788:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496795:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204555.496800:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204555.496807:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204555.496814:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.496819:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204555.496824:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204555.496831:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.496835:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204555.496840:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000020:00000040:1.0:1788204555.496845:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204555.496850:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204555.496853:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.496862:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204555.496866:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204555.496870:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204555.496876:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:1.0:1788204555.496882:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204555.496891:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205874432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204555.496905:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204555.496910:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204555.496919:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205874432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204555.496937:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204555.496941:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204555.496950:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204555.496952:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204555.496954:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204555.496957:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204555.496962:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6010 00000400:00000200:1.0:1788204555.496969:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204555.496976:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000200:3.1:1788204555.496980:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:1.0:1788204555.496982:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204555.496984:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1788204555.496990:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.496995:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.497000:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497007:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204555.497011:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204555.497015:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204555.497021:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204555.497024:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204555.497029:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204555.497035:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204555.497043:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497049:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.497054:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497060:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204555.497063:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204555.497071:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497077:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204555.497080:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204555.497085:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204555.497088:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497092:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204555.497095:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204555.497097:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497100:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204555.497107:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788204555.497114:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204555.497121:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204555.497125:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497129:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204555.497132:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204555.497136:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204555.497143:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788204555.497150:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204555.497161:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205874560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204555.497188:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204555.497193:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204555.497202:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205874560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204555.497231:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204555.497236:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204555.497241:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204555.497244:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204555.497247:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204555.497253:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204555.497258:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6010 00000400:00000200:1.0:1788204555.497267:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204555.497275:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204555.497281:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204555.497288:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497292:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.497298:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497302:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204555.497306:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204555.497309:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204555.497312:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204555.497316:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204555.497321:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204555.497326:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204555.497333:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497338:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204555.497342:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497347:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204555.497352:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204555.497357:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204555.497362:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204555.497392:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204555.497396:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204555.497400:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497405:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204555.497409:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204555.497414:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497418:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204555.497425:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204555.497430:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204555.497436:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204555.497441:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204555.497446:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204555.497450:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204555.497454:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204555.497460:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204555.497466:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204555.497477:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205874688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204555.497494:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204555.497496:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204555.497504:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205874688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204555.497523:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204555.497531:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6020) 00000800:00000200:3.1:1788204555.497590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204555.497592:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204555.497756:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204555.497759:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204555.498054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204555.498056:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204555.688502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204555.688508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204555.688511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204555.688512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204555.688514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204555.688515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204555.688517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204555.688517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204555.688530:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204555.688531:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204555.688540:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204556.008465:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204556.008472:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204556.008474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204556.008479:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204556.008482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204556.008486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204556.008489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204556.008491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204556.008494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204556.456502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204556.456509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204556.456511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204556.456516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204556.456520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204556.456523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204556.456525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204556.456528:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204556.456531:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204556.520487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204556.520495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204556.520498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204556.520504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204556.520509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204556.520513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204556.520531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204556.520535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204556.520539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204556.712481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204556.712488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204556.712493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204556.712495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204556.712498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204556.712499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204556.712502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204556.712504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204556.712560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204556.712563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204556.712566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204557.032453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204557.032460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204557.032462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204557.032466:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204557.032469:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204557.032472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204557.032475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204557.032477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204557.032480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204557.480457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204557.480465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204557.480467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204557.480472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204557.480475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204557.480479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204557.480482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204557.480485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204557.480488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204557.544454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204557.544461:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204557.544464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204557.544470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204557.544474:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204557.544478:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204557.544481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204557.544484:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204557.544488:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204557.544498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204557.544503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204557.544505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204557.544509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204557.544511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204557.544514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204557.544516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204557.544518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204557.544520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204557.736507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204557.736512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204557.736515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204557.736516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204557.736518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204557.736519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204557.736521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204557.736521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204557.736534:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204557.736535:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204557.736537:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204558.056459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204558.056465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204558.056466:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204558.056470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204558.056473:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204558.056476:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204558.056478:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204558.056480:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204558.056481:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204558.504483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204558.504490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204558.504492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204558.504497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204558.504501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204558.504505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204558.504508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204558.504510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204558.504513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204558.568474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204558.568474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204558.568483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204558.568483:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204558.568487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204558.568492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204558.568495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204558.568497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204558.568500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204558.568503:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204558.568505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204558.568507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204558.568509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204558.568511:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204558.568516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204558.568516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204558.568519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204558.568522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204558.760466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204558.760478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204558.760484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204558.760485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204558.760487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204558.760488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204558.760490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204558.760491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204558.760553:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204558.760557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204558.760560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204559.080472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204559.080480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204559.080482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204559.080487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204559.080490:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204559.080494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204559.080496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204559.080499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204559.080501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204559.528477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204559.528485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204559.528487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204559.528492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204559.528495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204559.528499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204559.528501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204559.528503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204559.528506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204559.592463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204559.592469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204559.592470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204559.592471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204559.592475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204559.592476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204559.592480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204559.592480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204559.592484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204559.592486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204559.592487:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204559.592490:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204559.592491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204559.592493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204559.592497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204559.592501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204559.592503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204559.592505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204559.784538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204559.784550:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204559.784552:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204559.784556:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784560:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204559.784563:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784565:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784568:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204559.784573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204559.784577:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204559.784579:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204559.784602:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204559.784605:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204559.784608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204560.104474:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300686849, at 4300687360 00000100:00000001:2.0:1788204560.104476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204560.104482:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300686849, at 4300687360 00000100:00000010:1.0:1788204560.104488:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000001:2.0:1788204560.104489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788204560.104491:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000001:2.0:1788204560.104493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204560.104498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204560.104501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204560.104503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204560.104505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204560.104508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204560.104511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.552466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204560.552472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204560.552474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204560.552478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204560.552480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204560.552483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204560.552485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204560.552487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204560.552488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204560.616480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788204560.616486:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204560.616487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204560.616491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204560.616495:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204560.616499:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788204560.616501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:1.0:1788204560.616502:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204560.616509:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204560.616509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204560.616515:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204560.616518:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6015 00000100:00000001:2.0:1788204560.616520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204560.616525:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204560.616528:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204560.616531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204560.616535:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204560.616538:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204560.616543:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616547:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616551:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616555:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204560.616559:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204560.616562:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204560.616565:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204560.616568:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204560.616570:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204560.616572:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204560.616575:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616577:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616579:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616581:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204560.616584:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204560.616587:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616589:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204560.616592:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204560.616595:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204560.616598:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616600:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204560.616602:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204560.616611:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204560.616614:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204560.616618:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204560.616620:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616623:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204560.616625:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204560.616630:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204560.616638:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788204560.616644:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204560.616651:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205874816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204560.616658:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204560.616661:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204560.616664:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205874816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204560.616681:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204560.616683:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204560.616685:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204560.616686:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204560.616687:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204560.616689:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204560.616692:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6015 00000400:00000200:1.0:1788204560.616695:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204560.616699:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204560.616702:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204560.616705:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616707:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616711:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616715:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204560.616719:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204560.616722:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204560.616726:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204560.616728:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204560.616734:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204560.616738:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204560.616745:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616747:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616759:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616764:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204560.616770:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204560.616777:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616782:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204560.616787:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204560.616795:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204560.616798:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616801:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204560.616805:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204560.616809:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616811:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204560.616816:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788204560.616822:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204560.616823:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204560.616824:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616825:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204560.616826:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204560.616828:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204560.616835:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788204560.616837:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204560.616846:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205874944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204560.616855:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204560.616858:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204560.616862:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205874944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204560.616886:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204560.616890:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204560.616894:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204560.616897:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204560.616898:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204560.616900:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204560.616904:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6015 00000400:00000200:1.0:1788204560.616910:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204560.616913:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204560.616916:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204560.616921:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616923:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616925:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616926:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204560.616929:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204560.616930:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204560.616934:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204560.616936:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204560.616937:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204560.616940:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204560.616946:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616947:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.616951:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616953:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204560.616955:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204560.616959:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204560.616963:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204560.616965:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204560.616968:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204560.616973:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616977:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204560.616980:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204560.616986:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.616990:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204560.616994:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000020:00000040:1.0:1788204560.616999:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204560.617004:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204560.617007:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.617011:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204560.617015:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204560.617017:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204560.617022:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788204560.617027:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204560.617035:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205875072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204560.617048:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204560.617049:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204560.617051:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204560.617056:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205875072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204560.617066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1788204560.617068:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204560.617072:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204560.617078:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204560.617082:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204560.617084:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204560.617088:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204560.617094:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6015 00000400:00000200:1.0:1788204560.617100:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204560.617105:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204560.617107:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204560.617110:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.617112:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.617117:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.617122:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204560.617136:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204560.617139:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204560.617141:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204560.617145:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204560.617149:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204560.617154:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204560.617161:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.617164:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204560.617170:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204560.617174:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204560.617180:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204560.617186:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204560.617191:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204560.617195:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204560.617199:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204560.617201:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.617205:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204560.617209:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204560.617214:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.617218:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204560.617223:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788204560.617227:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204560.617230:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204560.617231:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204560.617233:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204560.617234:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204560.617238:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204560.617244:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788204560.617250:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204560.617257:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205875200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204560.617273:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204560.617276:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204560.617284:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205875200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204560.617300:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204560.617307:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6025) 00000800:00000200:3.1:1788204560.617593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204560.617596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204560.617867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204560.617870:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204560.618158:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204560.618160:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00010000:00000001:1.0:1788204560.808464:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204560.808473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204560.808480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204560.808482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204560.808485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204560.808487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204560.808490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204560.808491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204560.808538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204560.808541:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204560.808544:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204561.128462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204561.128469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204561.128470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204561.128474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204561.128476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204561.128479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204561.128481:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204561.128483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204561.128485:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204561.576459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204561.576466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204561.576468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204561.576474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204561.576477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204561.576480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204561.576483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204561.576485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204561.576488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204561.640477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204561.640482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204561.640485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204561.640498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204561.640503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204561.640506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204561.640519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204561.640522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204561.640525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204561.832495:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204561.832497:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204561.832499:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832503:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204561.832504:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832505:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832507:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204561.832510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204561.832513:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204561.832514:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204561.832515:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832516:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204561.832516:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832517:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832518:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204561.832521:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204561.832522:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204561.832522:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832523:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204561.832524:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832525:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204561.832527:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204561.832543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204561.832545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204561.832546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204562.152472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204562.152479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204562.152481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204562.152486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204562.152489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204562.152492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204562.152495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204562.152497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204562.152500:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204562.600491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204562.600499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204562.600501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204562.600516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204562.600522:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204562.600526:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204562.600529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204562.600531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204562.600534:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.664464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204562.664471:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204562.664475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204562.664484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204562.664489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204562.664493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204562.664499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204562.664503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204562.664506:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204562.856473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204562.856480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204562.856486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204562.856487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204562.856490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204562.856492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204562.856495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204562.856496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204562.856549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204562.856551:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204562.856554:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204563.176474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204563.176480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204563.176482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204563.176485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204563.176488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204563.176491:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.176499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204563.176501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.176503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204563.624476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204563.624483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204563.624485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204563.624488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204563.624491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204563.624494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204563.624496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204563.624498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204563.624500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204563.688476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204563.688482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204563.688485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204563.688494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204563.688499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204563.688504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.688509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204563.688513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.688516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204563.880491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204563.880496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204563.880499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204563.880499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204563.880501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204563.880502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204563.880504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204563.880505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204563.880521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204563.880523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204563.880525:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204564.200456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204564.200462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204564.200464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204564.200468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204564.200470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204564.200473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204564.200475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204564.200477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204564.200480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204564.648493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204564.648499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204564.648502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204564.648507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204564.648511:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204564.648515:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204564.648517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204564.648520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204564.648523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204564.712469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204564.712475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204564.712477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204564.712483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204564.712487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204564.712491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204564.712494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204564.712497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204564.712500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204564.904455:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204564.904460:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204564.904464:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204564.904465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204564.904467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204564.904468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204564.904470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204564.904471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204564.904524:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204564.904527:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204564.904529:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204565.224462:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300691969, at 4300692480 00000100:00000001:2.0:1788204565.224463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204565.224467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204565.224470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204565.224473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204565.224474:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000001:2.0:1788204565.224476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204565.224480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204565.224482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204565.224484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204565.224486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204565.224554:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300691969, at 4300692480 00000100:00000010:1.0:1788204565.224560:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000001:1.0:1788204565.672478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204565.672483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204565.672485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204565.672490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204565.672493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204565.672496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204565.672497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204565.672499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204565.672501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204565.736473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204565.736476:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204565.736481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204565.736484:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204565.736485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736488:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788204565.736490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204565.736493:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204565.736496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204565.736498:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204565.736500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204565.736506:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6020 00000100:00000001:2.0:1788204565.736513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788204565.736518:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204565.736518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204565.736523:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204565.736524:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204565.736528:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204565.736531:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736533:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736535:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736536:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204565.736539:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204565.736540:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204565.736542:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204565.736544:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204565.736546:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204565.736548:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204565.736550:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736551:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736552:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736553:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204565.736555:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204565.736557:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736558:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204565.736560:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204565.736562:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204565.736564:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736565:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204565.736567:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204565.736574:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204565.736576:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204565.736579:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204565.736581:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736582:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204565.736584:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204565.736587:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204565.736593:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204565.736598:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204565.736604:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205875328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204565.736610:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204565.736614:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204565.736617:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205875328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204565.736630:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204565.736632:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204565.736634:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204565.736636:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204565.736637:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204565.736640:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204565.736644:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6020 00000400:00000200:0.0:1788204565.736648:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204565.736652:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204565.736657:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204565.736665:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736672:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736677:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736683:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204565.736686:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204565.736690:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204565.736694:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204565.736698:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204565.736703:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204565.736707:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204565.736713:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736715:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736718:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736720:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204565.736723:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204565.736728:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736730:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204565.736732:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204565.736738:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204565.736741:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736743:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204565.736748:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204565.736752:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736754:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204565.736759:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204565.736763:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204565.736768:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204565.736770:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736774:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204565.736776:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204565.736778:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204565.736783:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204565.736787:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204565.736793:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205875456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204565.736803:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204565.736805:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204565.736811:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205875456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204565.736817:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204565.736822:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204565.736825:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204565.736828:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204565.736831:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204565.736833:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204565.736838:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6020 00000400:00000200:0.0:1788204565.736846:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204565.736854:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204565.736859:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204565.736865:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736870:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736873:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736875:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204565.736878:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204565.736881:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204565.736884:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204565.736887:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204565.736890:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204565.736892:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204565.736897:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736901:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.736904:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736905:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204565.736907:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204565.736911:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204565.736913:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204565.736914:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204565.736916:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204565.736918:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736921:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204565.736924:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204565.736928:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736933:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204565.736937:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000010:0.0:1788204565.736938:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000800:00000200:3.1:1788204565.736940:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000020:00000040:0.0:1788204565.736941:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204565.736947:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204565.736949:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.736951:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204565.736954:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204565.736955:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204565.736958:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204565.736963:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204565.736968:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205875584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204565.736976:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204565.736979:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204565.736985:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205875584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204565.737004:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204565.737007:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204565.737010:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204565.737013:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204565.737014:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204565.737020:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204565.737024:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6020 00000400:00000200:0.0:1788204565.737030:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204565.737034:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204565.737047:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204565.737050:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.737055:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.737058:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.737060:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204565.737064:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204565.737066:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204565.737068:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204565.737071:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204565.737072:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204565.737074:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204565.737078:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.737080:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204565.737081:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204565.737082:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204565.737084:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204565.737089:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204565.737091:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204565.737092:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204565.737096:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204565.737100:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.737103:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204565.737107:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204565.737111:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.737114:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204565.737119:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204565.737124:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204565.737129:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204565.737132:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204565.737137:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204565.737141:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204565.737146:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204565.737153:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204565.737164:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204565.737174:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205875712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204565.737190:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204565.737194:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204565.737201:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205875712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204565.737211:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204565.737218:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6030) 00000800:00000200:3.1:1788204565.737379:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204565.737381:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204565.737598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204565.737600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204565.737777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204565.737779:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00010000:00000001:1.0:1788204565.928527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204565.928534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204565.928540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204565.928541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204565.928545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204565.928547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204565.928550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204565.928551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204565.928575:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204565.928578:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204565.928580:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204566.248439:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204566.248445:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204566.248447:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204566.248452:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204566.248455:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204566.248459:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.248462:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204566.248465:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.248468:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204566.696434:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204566.696440:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204566.696441:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204566.696444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204566.696446:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204566.696449:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204566.696450:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204566.696452:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204566.696454:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204566.760460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204566.760468:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204566.760472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204566.760478:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204566.760483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204566.760487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.760491:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204566.760494:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.760495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204566.952467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204566.952474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204566.952480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204566.952481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204566.952484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204566.952485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204566.952488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204566.952489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204566.952552:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204566.952556:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204566.952559:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204567.272486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204567.272493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204567.272495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.272500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204567.272503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204567.272507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204567.272509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204567.272511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204567.272514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204567.720499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204567.720506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204567.720508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204567.720513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204567.720516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204567.720520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.720522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204567.720525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.720527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.784441:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204567.784448:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204567.784450:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.784455:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204567.784458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204567.784459:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204567.784464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204567.784465:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.784468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.784470:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1788204567.784472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204567.784474:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.784477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204567.784480:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204567.784481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.784486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204567.784488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204567.784490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204567.976492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204567.976498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204567.976501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204567.976502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204567.976504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204567.976505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204567.976506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204567.976507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204567.976524:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204567.976525:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204567.976527:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204568.296433:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204568.296440:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204568.296442:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204568.296447:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204568.296450:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204568.296454:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.296456:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204568.296458:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.296461:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204568.744467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204568.744475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204568.744477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204568.744482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204568.744485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204568.744489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204568.744490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204568.744492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204568.744494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204568.808460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204568.808460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204568.808465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204568.808466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204568.808468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204568.808469:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204568.808474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204568.808475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204568.808478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204568.808479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204568.808483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.808483:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204568.808487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204568.808490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204568.808492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204568.808493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204568.808496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.808498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204569.000469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204569.000474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204569.000478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204569.000479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204569.000481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204569.000482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204569.000484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204569.000485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204569.000540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204569.000544:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204569.000547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204569.320436:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204569.320441:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204569.320443:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204569.320446:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204569.320448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204569.320451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.320453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204569.320454:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.320456:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204569.768477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204569.768482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204569.768484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204569.768488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204569.768490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204569.768493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204569.768495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204569.768497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204569.768500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.832423:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204569.832424:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204569.832429:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204569.832430:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204569.832431:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.832433:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204569.832436:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204569.832439:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204569.832440:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204569.832444:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.832445:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204569.832447:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204569.832449:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204569.832450:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204569.832454:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.832454:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204569.832457:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.832459:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204570.024500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204570.024506:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204570.024508:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204570.024511:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024513:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204570.024514:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024515:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024517:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204570.024520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204570.024522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204570.024523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204570.024539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204570.024541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204570.024544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204570.344454:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300697089, at 4300697600 00000100:00000040:1.0:1788204570.344454:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300697089, at 4300697600 00000100:00000010:1.0:1788204570.344468:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000010:0.0:1788204570.344469:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000001:2.0:1788204570.344558:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204570.344562:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204570.344564:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204570.344568:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204570.344571:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204570.344574:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204570.344576:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204570.344579:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204570.344582:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204570.792450:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204570.792456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204570.792457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204570.792461:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204570.792463:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204570.792466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204570.792467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204570.792470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204570.792471:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204570.856464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204570.856464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204570.856470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204570.856471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204570.856473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204570.856475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204570.856478:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204570.856480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204570.856482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204570.856485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204570.856486:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204570.856490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204570.856490:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204570.856493:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204570.856497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204570.856497:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204570.856500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204570.856503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204570.856590:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204570.856596:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204570.856598:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204570.856601:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204570.856605:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204570.856611:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6025 00000400:00000200:0.0:1788204570.856618:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204570.856624:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204570.856626:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204570.856628:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856630:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.856632:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856633:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204570.856636:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204570.856637:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204570.856639:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204570.856642:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204570.856643:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204570.856644:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204570.856646:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856647:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.856648:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856649:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204570.856651:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204570.856653:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856654:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204570.856656:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204570.856658:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204570.856660:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.856661:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204570.856663:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204570.856669:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204570.856671:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204570.856674:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204570.856675:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.856677:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204570.856679:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204570.856682:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204570.856689:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204570.856693:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204570.856699:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205875840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204570.856706:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204570.856709:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204570.856713:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205875840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204570.856734:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204570.856737:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204570.856740:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204570.856742:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204570.856743:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204570.856746:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204570.856749:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6025 00000400:00000200:0.0:1788204570.856753:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204570.856756:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204570.856758:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204570.856761:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856764:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.856769:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856774:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204570.856779:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204570.856782:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204570.856786:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204570.856790:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204570.856796:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204570.856801:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204570.856811:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856816:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.856823:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856829:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204570.856834:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204570.856842:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204570.856848:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204570.856852:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204570.856858:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204570.856862:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.856868:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204570.856875:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204570.856882:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.856890:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204570.856897:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204570.856902:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204570.856908:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204570.856911:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.856916:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204570.856919:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204570.856922:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204570.856927:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204570.856933:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204570.856942:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205875968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204570.856972:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204570.856975:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204570.856984:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205875968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204570.857001:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204570.857006:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204570.857015:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.1:1788204570.857016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204570.857018:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788204570.857019:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00080000:0.0:1788204570.857021:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204570.857026:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204570.857030:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6025 00000400:00000200:0.0:1788204570.857037:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204570.857043:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204570.857046:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204570.857051:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857054:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.857057:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857061:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204570.857064:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204570.857066:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204570.857069:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204570.857072:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204570.857074:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204570.857078:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204570.857083:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857087:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.857090:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857094:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204570.857098:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204570.857103:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857105:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204570.857110:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204570.857112:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204570.857116:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857120:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204570.857122:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204570.857123:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857127:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204570.857130:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204570.857132:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204570.857134:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204570.857137:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857138:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204570.857140:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204570.857142:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204570.857147:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204570.857151:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204570.857157:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205876096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204570.857174:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204570.857178:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204570.857186:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205876096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204570.857213:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204570.857215:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204570.857219:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204570.857223:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204570.857225:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204570.857229:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204570.857234:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6025 00000400:00000200:0.0:1788204570.857239:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204570.857244:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204570.857248:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204570.857254:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857259:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.857263:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857267:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204570.857270:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204570.857272:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204570.857276:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204570.857280:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204570.857285:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204570.857291:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204570.857299:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857301:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204570.857303:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857307:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204570.857313:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204570.857317:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204570.857322:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204570.857326:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204570.857330:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204570.857334:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857338:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204570.857342:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204570.857348:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857352:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204570.857358:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204570.857425:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204570.857431:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204570.857435:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204570.857441:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204570.857443:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204570.857446:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204570.857452:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204570.857457:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204570.857464:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205876224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204570.857474:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204570.857478:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204570.857485:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205876224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204570.857502:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204570.857507:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6035) 00000800:00000200:3.1:1788204570.857594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204570.857596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204570.857831:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204570.857832:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204570.858053:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204570.858054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00010000:00000001:1.0:1788204571.048459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204571.048465:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204571.048467:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204571.048469:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048473:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204571.048474:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048476:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048477:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204571.048480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204571.048483:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204571.048484:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204571.048485:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048485:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204571.048486:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048487:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048488:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204571.048491:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204571.048492:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204571.048493:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048494:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204571.048495:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048496:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204571.048497:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204571.048573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204571.048576:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204571.048579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204571.368457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204571.368462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204571.368464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204571.368469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204571.368472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204571.368475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.368476:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204571.368478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.368480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204571.816482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204571.816492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204571.816494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204571.816501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204571.816505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204571.816510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204571.816513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204571.816516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204571.816520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204571.880496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204571.880503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204571.880506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204571.880509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204571.880511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204571.880512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204571.880519:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204571.880521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204571.880523:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204571.880528:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204571.880529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204571.880534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.880536:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204571.880540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.880544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204571.880545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204571.880548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204571.880551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204572.072518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204572.072526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204572.072532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204572.072533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204572.072535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204572.072536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204572.072539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204572.072540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204572.072556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204572.072558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204572.072560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204572.392461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204572.392466:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204572.392468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204572.392472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204572.392474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204572.392478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.392481:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204572.392483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.392486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204572.840465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204572.840471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204572.840473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204572.840476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204572.840478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204572.840481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204572.840482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204572.840484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204572.840486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204572.904461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204572.904461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204572.904469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204572.904469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204572.904473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204572.904474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204572.904481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204572.904482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204572.904488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204572.904488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204572.904491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.904493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204572.904495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204572.904499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204572.904502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204572.904509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204572.904512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.904515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204573.096452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204573.096467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204573.096474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204573.096475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204573.096477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204573.096478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204573.096480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204573.096480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204573.096524:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204573.096528:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204573.096531:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204573.416472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204573.416479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204573.416481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204573.416486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204573.416489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204573.416492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204573.416495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204573.416498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204573.416501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204573.864492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204573.864499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204573.864501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204573.864507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204573.864510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204573.864513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204573.864515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204573.864517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204573.864520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204573.928510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204573.928519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204573.928523:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204573.928526:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1788204573.928530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204573.928532:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204573.928536:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204573.928541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204573.928542:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204573.928546:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204573.928548:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204573.928552:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204573.928553:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204573.928557:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204573.928561:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204573.928565:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204573.928572:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204573.928576:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204574.120561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204574.120575:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204574.120582:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204574.120583:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204574.120588:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204574.120589:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204574.120593:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204574.120594:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204574.120616:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204574.120619:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204574.120622:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204574.440466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204574.440473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204574.440475:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204574.440480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204574.440484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204574.440490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204574.440493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204574.440496:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204574.440499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204574.888480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204574.888487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204574.888489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204574.888494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204574.888497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204574.888501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204574.888503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204574.888506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204574.888509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204574.952454:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204574.952454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204574.952461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204574.952462:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204574.952464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204574.952466:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204574.952472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204574.952472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204574.952476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204574.952477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204574.952480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204574.952482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204574.952483:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204574.952488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204574.952492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204574.952494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204574.952497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204574.952500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204575.144465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204575.144472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204575.144478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204575.144479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204575.144483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204575.144484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204575.144487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204575.144489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204575.144551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204575.144554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204575.144556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204575.464486:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300702209, at 4300702720 00000100:00000040:1.0:1788204575.464486:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300702209, at 4300702720 00000100:00000001:2.0:1788204575.464494:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:0.0:1788204575.464500:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000010:1.0:1788204575.464502:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000001:2.0:1788204575.464502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204575.464507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204575.464513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204575.464518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204575.464521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204575.464524:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204575.464526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204575.464529:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204575.912472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204575.912479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204575.912481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204575.912486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204575.912490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204575.912494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204575.912496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204575.912498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204575.912500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204575.976481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204575.976487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204575.976489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204575.976493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204575.976497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204575.976501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204575.976503:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204575.976506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204575.976506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204575.976511:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204575.976512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204575.976516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204575.976517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976518:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204575.976522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204575.976525:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204575.976529:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204575.976532:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204575.976536:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204575.976542:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204575.976543:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6030 00000100:00000001:2.0:1788204575.976549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204575.976554:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204575.976557:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204575.976560:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204575.976568:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204575.976573:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204575.976576:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976579:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.976582:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976585:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204575.976589:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204575.976592:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204575.976595:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204575.976598:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204575.976601:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204575.976603:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204575.976606:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976608:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.976610:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976612:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204575.976615:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204575.976619:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976622:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204575.976625:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204575.976628:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204575.976631:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976633:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204575.976636:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204575.976645:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204575.976649:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204575.976653:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204575.976655:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976658:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204575.976660:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204575.976664:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204575.976671:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204575.976677:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204575.976686:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205876352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204575.976696:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204575.976699:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204575.976704:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205876352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204575.976725:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204575.976728:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204575.976731:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204575.976732:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204575.976734:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204575.976737:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204575.976739:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6030 00000400:00000200:0.0:1788204575.976743:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204575.976747:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204575.976750:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204575.976753:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976755:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.976758:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976762:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204575.976765:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204575.976769:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204575.976772:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204575.976775:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204575.976778:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204575.976783:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204575.976791:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976796:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.976801:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976805:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204575.976809:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204575.976815:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204575.976820:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204575.976824:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204575.976829:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204575.976833:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976835:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204575.976840:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204575.976846:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976849:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204575.976856:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204575.976872:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204575.976875:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204575.976877:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.976879:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204575.976887:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204575.976889:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204575.976896:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204575.976902:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204575.976912:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205876480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204575.976928:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204575.976933:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204575.976939:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205876480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204575.976953:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204575.976959:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204575.976965:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204575.976969:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204575.976972:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204575.976977:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204575.976989:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6030 00000400:00000200:0.0:1788204575.977009:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204575.977015:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204575.977018:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204575.977023:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977027:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204575.977031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204575.977033:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977036:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204575.977038:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204575.977040:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204575.977041:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204575.977043:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204575.977047:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204575.977049:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204575.977053:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977055:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.977057:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977061:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204575.977067:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204575.977073:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977078:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204575.977082:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204575.977085:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204575.977087:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977090:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204575.977095:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204575.977102:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977107:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204575.977113:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204575.977119:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204575.977125:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204575.977127:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977130:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204575.977133:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204575.977138:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204575.977146:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204575.977151:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204575.977162:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205876608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204575.977180:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204575.977185:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204575.977194:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205876608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204575.977211:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204575.977214:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204575.977219:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204575.977223:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204575.977225:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204575.977230:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204575.977234:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6030 00000400:00000200:0.0:1788204575.977242:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204575.977249:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204575.977255:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204575.977262:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977268:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.977274:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977276:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204575.977280:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204575.977282:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204575.977286:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204575.977290:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204575.977296:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204575.977300:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204575.977307:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977313:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204575.977318:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977322:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204575.977327:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204575.977334:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204575.977340:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204575.977344:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204575.977348:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204575.977353:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977357:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204575.977360:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204575.977383:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977385:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204575.977389:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204575.977392:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204575.977395:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204575.977396:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204575.977398:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204575.977399:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204575.977403:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204575.977409:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204575.977417:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204575.977423:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205876736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204575.977429:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204575.977433:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204575.977441:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205876736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204575.977454:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204575.977460:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204575.977461:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6040) 00000800:00000200:3.1:1788204575.977640:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204575.977762:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00010000:00000001:1.0:1788204576.168519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204576.168525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204576.168529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204576.168531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204576.168534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204576.168535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204576.168538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204576.168539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204576.168555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204576.168557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204576.168560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204576.488471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204576.488479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204576.488481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204576.488485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204576.488488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204576.488492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204576.488495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204576.488497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204576.488499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204576.936474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204576.936486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204576.936488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204576.936491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204576.936493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204576.936496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204576.936498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204576.936500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204576.936502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.000471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204577.000480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204577.000482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.000487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204577.000492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204577.000496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204577.000500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204577.000504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204577.000509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204577.000540:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204577.000546:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204577.000547:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204577.000550:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204577.000553:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204577.000556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204577.000558:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204577.000559:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204577.000561:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204577.192473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204577.192480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204577.192486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204577.192488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204577.192491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204577.192492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204577.192495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204577.192497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204577.192555:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204577.192559:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204577.192563:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204577.512512:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204577.512522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204577.512525:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.512531:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204577.512535:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204577.512540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204577.512543:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204577.512547:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204577.512551:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204577.960492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204577.960500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204577.960502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204577.960507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204577.960510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204577.960514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204577.960516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204577.960519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204577.960522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204578.024504:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204578.024512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204578.024514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204578.024525:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204578.024529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204578.024533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.024536:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204578.024539:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.024542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204578.024572:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204578.024579:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204578.024581:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204578.024586:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204578.024590:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204578.024595:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204578.024598:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204578.024600:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204578.024602:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204578.216552:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204578.216558:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204578.216563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204578.216564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204578.216568:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204578.216569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204578.216572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204578.216573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204578.216593:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204578.216596:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204578.216599:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204578.536453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204578.536463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204578.536465:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204578.536471:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204578.536476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204578.536479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.536482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204578.536485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.536488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204578.984466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204578.984474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204578.984477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204578.984482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204578.984486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204578.984491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204578.984494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204578.984497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204578.984500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204579.048433:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204579.048443:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204579.048445:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204579.048451:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204579.048454:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204579.048458:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.048460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204579.048462:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.048464:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204579.048592:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204579.048596:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204579.048598:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204579.048601:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204579.048604:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204579.048606:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204579.048608:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204579.048609:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204579.048611:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204579.048706:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:0.0:1788204579.048757:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204579.048769:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204579.048772:0:2333:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:0.0:1788204579.048777:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204579.048781:0:2333:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788204579.048785:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204579.048789:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:0.0:1788204579.048793:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204579.048797:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204579.048802:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f42a8 00000400:00000010:0.0:1788204579.048943:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f42a8. 00000100:00000001:0.0:1788204579.048951:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204579.048965:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ec380 x1875062205876352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049004:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204579.049009:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205876352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049018:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204579.049021:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204579.049034:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204579.049037:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecb00. 00000400:00000200:0.0:1788204579.049040:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204579.049043:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204579.049046:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4a18 00000400:00000010:0.0:1788204579.049048:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4a18. 00000100:00000001:0.0:1788204579.049051:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204579.049055:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ed500 x1875062205876480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049062:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204579.049065:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205876480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049072:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204579.049074:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204579.049079:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204579.049081:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ece00. 00000400:00000200:0.0:1788204579.049084:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204579.049087:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204579.049090:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4b28 00000400:00000010:0.0:1788204579.049091:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4b28. 00000100:00000001:0.0:1788204579.049094:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204579.049098:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105edc00 x1875062205876608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049104:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204579.049107:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205876608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049113:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204579.049114:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204579.049119:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204579.049121:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:0.0:1788204579.049123:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204579.049126:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204579.049128:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4880 00000400:00000010:0.0:1788204579.049130:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4880. 00000100:00000001:0.0:1788204579.049132:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204579.049136:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ef480 x1875062205876736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049143:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204579.049146:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205876736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204579.049151:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204579.049153:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1788204579.240483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204579.240490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204579.240497:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204579.240500:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204579.240504:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240506:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204579.240508:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240510:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240512:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204579.240516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204579.240519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204579.240521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204579.240587:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204579.240592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204579.240595:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204579.560505:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204579.560515:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204579.560517:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204579.560528:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204579.560531:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204579.560536:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.560538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204579.560540:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.560542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204580.008488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204580.008496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204580.008498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204580.008503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204580.008507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204580.008512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204580.008515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204580.008518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204580.008521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204580.072504:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204580.072515:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204580.072517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204580.072530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204580.072535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204580.072540:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.072544:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204580.072547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.072551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204580.072562:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204580.072567:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204580.072579:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204580.072583:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204580.072586:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204580.072589:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204580.072591:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204580.072593:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204580.072595:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204580.264541:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204580.264548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204580.264553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204580.264554:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204580.264557:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204580.264558:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204580.264560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204580.264561:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204580.264579:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204580.264582:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204580.264584:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204580.584443:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300707329, at 4300707840 00000100:00000010:0.0:1788204580.584456:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00000040:1.0:1788204580.584459:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300707329, at 4300707840 00000100:00000001:2.0:1788204580.584468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:1.0:1788204580.584471:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000001:2.0:1788204580.584477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204580.584481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204580.584485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204580.584489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204580.584492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.584495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204580.584497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.584501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204581.032484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204581.032489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204581.032491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204581.032494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204581.032497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204581.032500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204581.032501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204581.032503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204581.032505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.096499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204581.096507:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204581.096510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.096516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204581.096521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204581.096525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.096528:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.096532:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.096536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204581.096553:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204581.096560:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204581.096564:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204581.096569:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204581.096575:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204581.096579:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204581.096582:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204581.096584:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204581.096588:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.096802:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204581.096809:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204581.096814:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204581.096830:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.096855:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.096859:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.096867:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204581.096873:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204581.096890:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.096900:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205876864:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204581.096912:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204581.096921:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204581.096926:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.096931:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204581.096946:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204581.096956:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204581.096970:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204581.096981:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204581.096993:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204581.097016:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205876864, portal 25 00000100:00000001:2.0:1788204581.097028:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204581.097032:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:2.0:1788204581.097048:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097081:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204581.097089:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204581.097097:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204581.097108:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205876864, offset 0 00000400:00000200:2.0:1788204581.097127:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204581.097144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.097158:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204581.097168:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204581.097182:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204581.097192:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204581.097202:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204581.097214:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58706:58706:256:4294967295] 192.168.201.151@tcp LPNI seq info [58686:58686:8:4294967295] 00000400:00000200:2.0:1788204581.097231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.097258:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204581.097272:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.097280:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204581.097294:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204581.097301:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204581.097317:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204581.097326:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204581.097341:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.097347:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204581.097359:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204581.097410:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204581.097419:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204581.097423:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204581.097429:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204581.097436:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097448:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.097450:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097457:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.097458:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204581.097465:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204581.097471:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204581.097479:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300712450 00000100:00000010:2.0:1788204581.097490:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1f60. 00000100:00000001:2.0:1788204581.097504:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204581.097531:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300712450 00000100:00000010:2.0:1788204581.097539:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1d20. 00000100:00000001:2.0:1788204581.097546:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204581.097553:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204581.097562:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.097569:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204581.097581:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204581.097616:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204581.097621:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204581.097626:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204581.097632:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204581.097637:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204581.097645:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204581.097654:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204581.097681:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097701:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204581.097705:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.097712:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.097716:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204581.097728:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097748:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204581.097759:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097817:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204581.097825:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.097828:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.097833:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.097840:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204581.097844:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204581.097855:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204581.097868:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204581.097876:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204581.097890:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204581.097920:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205876864:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204581.097939:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.097947:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.097956:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205876864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.097974:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204581.097978:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204581.097982:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204581.097987:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204581.097990:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204581.097993:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204581.097996:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204581.097999:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204581.098003:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204581.098006:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204581.098007:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:2.0:1788204581.098012:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204581.098013:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098016:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098030:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204581.098032:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098033:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204581.098035:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204581.098036:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204581.098040:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098046:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.098048:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098050:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204581.098053:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204581.098056:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.098059:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205876992:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204581.098062:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204581.098065:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204581.098067:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098069:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204581.098073:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204581.098075:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204581.098080:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204581.098084:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204581.098088:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204581.098092:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205876992, portal 10 00000100:00000001:2.0:1788204581.098094:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204581.098095:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:2.0:1788204581.098103:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098110:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204581.098112:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204581.098114:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204581.098116:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205876992, offset 0 00000400:00000200:2.0:1788204581.098121:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098127:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098132:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204581.098135:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204581.098137:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204581.098140:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204581.098141:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098144:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58707:58707:256:4294967295] 192.168.201.151@tcp LPNI seq info [58687:58687:8:4294967295] 00000400:00000200:2.0:1788204581.098147:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098154:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204581.098158:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098161:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204581.098164:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204581.098167:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204581.098170:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204581.098172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204581.098174:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098176:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204581.098179:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204581.098182:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204581.098185:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204581.098187:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204581.098189:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204581.098193:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098201:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.098203:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098210:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098212:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204581.098215:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204581.098219:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204581.098222:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204581.098224:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300708507 00000100:00000001:2.0:1788204581.098227:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098230:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300708507 00000100:00000001:2.0:1788204581.098232:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098234:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204581.098236:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098238:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204581.098242:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204581.098250:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204581.098252:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204581.098253:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204581.098256:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204581.098257:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204581.098260:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204581.098263:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204581.098271:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098277:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204581.098279:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098281:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098282:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204581.098286:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098292:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204581.098296:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098303:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204581.098306:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098308:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098309:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098311:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204581.098314:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204581.098317:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204581.098320:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204581.098323:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204581.098326:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204581.098335:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205876992:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204581.098338:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204581.098343:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098349:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.098351:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098353:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204581.098355:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204581.098357:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.098360:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877120:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204581.098388:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204581.098391:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204581.098392:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098394:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204581.098398:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204581.098400:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204581.098404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204581.098407:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204581.098411:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204581.098414:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877120, portal 4 00000100:00000001:2.0:1788204581.098416:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204581.098417:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:2.0:1788204581.098423:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098430:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204581.098432:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204581.098434:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204581.098437:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205877120, offset 0 00000400:00000200:2.0:1788204581.098441:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098445:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098448:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204581.098451:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204581.098454:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204581.098456:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204581.098458:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098462:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58708:58708:256:4294967295] 192.168.201.151@tcp LPNI seq info [58688:58688:8:4294967295] 00000400:00000200:2.0:1788204581.098466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098472:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204581.098477:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098480:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204581.098484:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204581.098487:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204581.098491:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204581.098494:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204581.098498:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098499:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204581.098503:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204581.098506:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204581.098509:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204581.098510:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204581.098513:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204581.098516:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098524:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.098527:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098534:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098536:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204581.098539:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204581.098564:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204581.098567:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204581.098569:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300708394 00000100:00000001:2.0:1788204581.098572:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098574:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300708394 00000100:00000001:2.0:1788204581.098576:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098579:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204581.098581:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098583:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204581.098587:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204581.098595:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204581.098597:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204581.098598:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204581.098601:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204581.098603:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204581.098605:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204581.098608:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204581.098617:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098624:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204581.098625:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098627:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098628:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204581.098632:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098638:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204581.098642:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098649:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204581.098651:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098653:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098654:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098656:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204581.098658:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204581.098661:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204581.098665:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204581.098667:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204581.098671:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204581.098679:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877120:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204581.098683:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204581.098687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098694:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.098695:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098698:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204581.098700:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204581.098702:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.098705:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204581.098708:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204581.098710:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204581.098712:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204581.098714:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204581.098718:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204581.098720:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204581.098724:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204581.098727:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204581.098732:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204581.098735:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877248, portal 4 00000100:00000001:2.0:1788204581.098737:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204581.098740:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:2.0:1788204581.098747:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098754:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204581.098756:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204581.098759:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204581.098761:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205877248, offset 0 00000400:00000200:2.0:1788204581.098765:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098782:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098785:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204581.098787:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204581.098789:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204581.098791:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204581.098792:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204581.098794:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58709:58709:256:4294967295] 192.168.201.151@tcp LPNI seq info [58689:58689:8:4294967295] 00000400:00000200:2.0:1788204581.098797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204581.098802:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204581.098805:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098808:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204581.098812:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204581.098815:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204581.098820:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204581.098823:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204581.098828:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204581.098831:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204581.098834:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204581.098837:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204581.098840:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204581.098842:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204581.098844:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204581.098848:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098856:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.098859:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098865:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098867:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204581.098870:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204581.098872:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204581.098876:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204581.098878:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300708400 00000100:00000001:2.0:1788204581.098880:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098883:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300708400 00000100:00000001:2.0:1788204581.098885:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300712450 00000100:00000001:2.0:1788204581.098888:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204581.098890:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.098893:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204581.098896:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204581.098905:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204581.098906:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204581.098908:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204581.098911:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204581.098913:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204581.098916:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204581.098919:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204581.098928:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098935:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204581.098936:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098938:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098940:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204581.098943:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098950:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204581.098954:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.098962:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204581.098964:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098966:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204581.098967:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098969:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204581.098971:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204581.098973:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204581.098976:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204581.098978:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204581.098981:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204581.098989:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204581.098993:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.098996:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.098999:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205876992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.099006:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204581.099007:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204581.099009:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204581.099012:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204581.099013:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204581.099015:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204581.099018:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204581.099019:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204581.099022:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204581.099023:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204581.099024:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:2.0:1788204581.099026:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204581.099028:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.099029:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.099033:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205877120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.099039:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204581.099040:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204581.099042:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204581.099044:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204581.099045:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204581.099046:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204581.099047:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204581.099048:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:2.0:1788204581.099051:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204581.099052:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204581.099053:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:2.0:1788204581.099055:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204581.099056:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.099057:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204581.099060:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205877248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204581.099067:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204581.099069:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204581.099070:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204581.099072:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204581.099074:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204581.099075:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204581.099077:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204581.099078:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204581.099080:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204581.099081:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204581.099083:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204581.099085:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204581.099086:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.099087:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.099091:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204581.099093:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.099094:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204581.099096:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204581.288465:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204581.288467:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204581.288470:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288474:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204581.288475:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288477:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288478:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204581.288481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204581.288484:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204581.288485:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204581.288486:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288487:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204581.288488:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288489:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288492:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204581.288496:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204581.288497:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204581.288499:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288500:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204581.288501:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204581.288504:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204581.288568:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204581.288574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204581.288577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204581.608510:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204581.608517:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204581.608519:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204581.608524:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204581.608527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204581.608531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.608534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204581.608536:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204581.608538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204582.056440:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204582.056447:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204582.056449:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204582.056453:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204582.056456:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204582.056460:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204582.056462:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204582.056463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204582.056466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204582.120502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204582.120511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204582.120514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204582.120521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204582.120525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204582.120530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204582.120533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204582.120537:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204582.120540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204582.120571:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204582.120574:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204582.120575:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204582.120578:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204582.120580:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204582.120582:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204582.120584:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204582.120585:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204582.120587:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204582.312525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204582.312532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204582.312537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204582.312538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204582.312540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204582.312541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204582.312543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204582.312544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204582.312560:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204582.312562:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204582.312564:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204582.632503:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204582.632511:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204582.632513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204582.632519:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204582.632522:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204582.632526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204582.632529:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204582.632533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204582.632536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204583.080495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204583.080502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204583.080504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204583.080509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204583.080512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204583.080516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204583.080518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204583.080520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204583.080522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204583.144481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204583.144481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204583.144488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204583.144489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204583.144491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204583.144493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204583.144499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204583.144500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204583.144504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204583.144505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204583.144508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204583.144510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204583.144511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204583.144515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204583.144518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204583.144522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204583.144525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204583.144527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204583.336496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204583.336504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204583.336510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204583.336511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204583.336513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204583.336515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204583.336519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204583.336519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204583.336573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204583.336577:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204583.336580:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204583.656485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204583.656493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204583.656496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204583.656502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204583.656506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204583.656511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204583.656514:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204583.656517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204583.656520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204584.104490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204584.104500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204584.104503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204584.104510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204584.104515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204584.104520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204584.104523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204584.104526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204584.104529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204584.168476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204584.168476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204584.168485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204584.168485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204584.168489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204584.168489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204584.168497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204584.168498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204584.168504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204584.168504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204584.168510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204584.168510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204584.168516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204584.168518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204584.168521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204584.168521:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204584.168524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204584.168526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204584.360521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204584.360528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204584.360533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204584.360534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204584.360536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204584.360537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204584.360540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204584.360540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204584.360558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204584.360566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204584.360568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204584.680518:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204584.680527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204584.680529:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204584.680534:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204584.680538:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204584.680543:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204584.680545:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204584.680548:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204584.680550:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204585.128461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204585.128469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204585.128472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204585.128477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204585.128481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204585.128484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204585.128487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204585.128489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204585.128492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204585.192494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204585.192494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204585.192504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204585.192505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204585.192509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204585.192510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204585.192523:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204585.192524:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204585.192535:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204585.192535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204585.192545:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204585.192546:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204585.192557:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204585.192560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204585.192564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204585.192564:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204585.192568:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204585.192572:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204585.384476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204585.384484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204585.384489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204585.384491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204585.384494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204585.384494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204585.384498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204585.384498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204585.384554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204585.384558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204585.384561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204585.704454:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300712450, at 4300712960 00000100:00000040:1.0:1788204585.704454:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300712450, at 4300712960 00000100:00000010:0.0:1788204585.704474:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000010:1.0:1788204585.704476:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000001:1.0:1788204586.152445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204586.152454:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204586.152457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204586.152463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204586.152468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204586.152472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204586.152475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204586.152478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204586.152481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204586.216520:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204586.216520:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204586.216531:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204586.216535:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204586.216536:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204586.216539:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.216546:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204586.216547:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204586.216553:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204586.216553:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204586.216560:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204586.216561:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.216568:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.216572:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204586.216576:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.216577:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204586.216581:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204586.216587:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.216807:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204586.216812:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204586.216816:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204586.216823:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.216831:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.216834:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.216838:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204586.216841:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204586.216844:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.216847:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205877376:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204586.216854:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204586.216859:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204586.216863:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.216867:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204586.216873:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204586.216877:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204586.216884:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204586.216888:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204586.216909:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204586.216919:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877376, portal 25 00000100:00000001:2.0:1788204586.216925:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204586.216927:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000040:2.0:1788204586.216933:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.216943:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204586.216948:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204586.216953:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204586.216958:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205877376, offset 0 00000400:00000200:2.0:1788204586.216967:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204586.216982:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.216993:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204586.217003:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204586.217012:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204586.217019:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204586.217025:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217031:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58710:58710:256:4294967295] 192.168.201.151@tcp LPNI seq info [58690:58690:8:4294967295] 00000400:00000200:2.0:1788204586.217041:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217056:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204586.217076:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217086:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204586.217099:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204586.217106:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204586.217117:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204586.217122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204586.217131:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217135:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204586.217145:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204586.217152:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204586.217159:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204586.217164:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204586.217170:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204586.217177:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217196:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.217202:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217215:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217219:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204586.217226:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204586.217234:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204586.217238:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300717569 00000100:00000010:2.0:1788204586.217246:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ea0. 00000100:00000001:2.0:1788204586.217258:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204586.217279:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300717569 00000100:00000010:2.0:1788204586.217287:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c00. 00000100:00000001:2.0:1788204586.217294:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204586.217300:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204586.217303:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217305:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204586.217311:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204586.217318:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204586.217319:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204586.217321:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204586.217323:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204586.217324:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204586.217336:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204586.217338:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204586.217345:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217350:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204586.217351:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217354:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217355:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204586.217359:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217386:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204586.217392:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217399:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204586.217402:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217404:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217405:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217408:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204586.217410:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204586.217413:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204586.217417:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204586.217421:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204586.217426:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204586.217434:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205877376:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204586.217447:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217449:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.217453:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205877376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217465:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204586.217468:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204586.217470:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204586.217474:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204586.217476:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204586.217478:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204586.217480:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204586.217482:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204586.217486:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204586.217488:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204586.217489:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:2.0:1788204586.217491:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204586.217492:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217494:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217501:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204586.217502:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217504:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204586.217505:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204586.217506:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204586.217508:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217513:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.217514:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217515:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204586.217517:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204586.217518:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.217520:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877504:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204586.217522:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204586.217524:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204586.217525:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217526:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204586.217528:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204586.217529:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204586.217532:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204586.217535:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204586.217537:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204586.217539:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877504, portal 10 00000100:00000001:2.0:1788204586.217540:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204586.217541:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:2.0:1788204586.217546:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217550:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204586.217552:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204586.217553:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204586.217555:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205877504, offset 0 00000400:00000200:2.0:1788204586.217557:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217559:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217561:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204586.217564:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204586.217565:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204586.217567:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204586.217569:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217571:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58711:58711:256:4294967295] 192.168.201.151@tcp LPNI seq info [58691:58691:8:4294967295] 00000400:00000200:2.0:1788204586.217573:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217577:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204586.217580:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217582:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204586.217585:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204586.217586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204586.217590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204586.217591:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204586.217593:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217595:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204586.217597:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204586.217599:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204586.217601:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204586.217602:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204586.217604:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204586.217606:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217611:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.217613:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217617:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217618:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204586.217620:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204586.217623:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204586.217626:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204586.217628:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300713627 00000100:00000001:2.0:1788204586.217629:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.217632:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300713627 00000100:00000001:2.0:1788204586.217634:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.217636:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204586.217638:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217640:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204586.217643:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204586.217651:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204586.217652:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204586.217654:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204586.217656:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204586.217658:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204586.217660:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204586.217663:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204586.217671:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217696:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204586.217697:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217699:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217700:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204586.217704:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217711:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204586.217715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217721:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204586.217724:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217726:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217727:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217729:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204586.217731:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204586.217733:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204586.217735:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204586.217737:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204586.217739:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204586.217744:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877504:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204586.217747:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204586.217749:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217753:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.217754:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217756:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204586.217757:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204586.217759:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.217761:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877632:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204586.217763:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204586.217764:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204586.217765:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217766:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204586.217768:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204586.217770:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204586.217772:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204586.217774:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204586.217777:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204586.217778:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877632, portal 4 00000100:00000001:2.0:1788204586.217780:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204586.217781:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:2.0:1788204586.217785:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217789:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204586.217790:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204586.217792:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204586.217793:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205877632, offset 0 00000400:00000200:2.0:1788204586.217795:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217798:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217800:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204586.217802:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204586.217804:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204586.217805:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204586.217807:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217809:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58712:58712:256:4294967295] 192.168.201.151@tcp LPNI seq info [58692:58692:8:4294967295] 00000400:00000200:2.0:1788204586.217811:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217815:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204586.217818:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217820:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204586.217822:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204586.217824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204586.217827:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204586.217829:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204586.217831:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.217832:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204586.217834:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204586.217837:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204586.217838:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204586.217839:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204586.217841:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204586.217843:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217848:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.217850:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217854:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217855:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204586.217857:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204586.217859:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204586.217861:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204586.217862:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300713513 00000100:00000001:2.0:1788204586.217864:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.217865:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300713513 00000100:00000001:2.0:1788204586.217867:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.217868:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204586.217869:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.217870:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204586.217872:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204586.217877:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204586.217878:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204586.217879:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204586.217880:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204586.217881:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204586.217882:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204586.217884:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204586.217888:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217892:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204586.217893:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217894:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217895:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204586.217897:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217900:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204586.217902:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217917:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204586.217918:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217919:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.217919:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.217920:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204586.217922:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204586.217923:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204586.217925:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204586.217927:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204586.217930:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204586.217935:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877632:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204586.217937:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204586.217939:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217942:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204586.217943:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217944:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204586.217945:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204586.217946:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.217948:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204586.217950:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204586.217951:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204586.217952:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204586.217953:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204586.217955:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204586.217956:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204586.217958:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204586.217959:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204586.217961:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204586.217963:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877760, portal 4 00000100:00000001:2.0:1788204586.217964:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204586.217965:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:2.0:1788204586.217969:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.217973:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204586.217974:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204586.217975:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204586.217977:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205877760, offset 0 00000400:00000200:2.0:1788204586.217982:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217985:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.217987:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204586.217990:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204586.217992:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204586.217994:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204586.217996:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204586.217999:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58713:58713:256:4294967295] 192.168.201.151@tcp LPNI seq info [58693:58693:8:4294967295] 00000400:00000200:2.0:1788204586.218004:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204586.218009:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204586.218014:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.218018:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204586.218021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204586.218024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204586.218028:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204586.218030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204586.218033:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204586.218034:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204586.218036:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204586.218038:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204586.218041:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204586.218042:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204586.218043:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204586.218046:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218050:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.218052:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218058:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.218059:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204586.218073:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204586.218075:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204586.218078:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204586.218079:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300713519 00000100:00000001:2.0:1788204586.218082:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.218084:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300713519 00000100:00000001:2.0:1788204586.218086:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300717569 00000100:00000001:2.0:1788204586.218088:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204586.218090:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.218092:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204586.218096:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204586.218103:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204586.218105:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204586.218106:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204586.218108:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204586.218110:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204586.218112:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204586.218116:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204586.218124:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218129:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204586.218130:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218131:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.218132:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204586.218134:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218139:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204586.218141:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218145:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204586.218147:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218148:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204586.218149:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218150:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204586.218151:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204586.218153:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204586.218154:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204586.218155:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204586.218158:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204586.218162:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205877760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204586.218164:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218166:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.218168:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205877504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218171:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204586.218172:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204586.218174:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204586.218175:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204586.218176:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204586.218177:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204586.218179:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204586.218180:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204586.218182:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204586.218182:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204586.218183:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:2.0:1788204586.218186:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204586.218186:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218187:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.218189:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205877632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218197:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204586.218198:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204586.218199:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204586.218200:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204586.218201:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204586.218201:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204586.218202:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204586.218203:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204586.218204:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204586.218205:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204586.218205:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:2.0:1788204586.218207:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204586.218208:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218208:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204586.218210:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205877760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204586.218213:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204586.218214:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204586.218214:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204586.218215:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204586.218216:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204586.218217:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204586.218217:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204586.218218:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204586.218219:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204586.218220:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204586.218220:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:2.0:1788204586.218221:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204586.218222:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218223:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204586.218225:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204586.218226:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.218227:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204586.218228:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204586.408506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204586.408512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204586.408518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204586.408519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204586.408522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204586.408523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204586.408526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204586.408528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204586.408544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204586.408547:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204586.408550:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204587.176458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204587.176463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204587.176465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204587.176468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204587.176470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204587.176473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204587.176474:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204587.176476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204587.176478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204587.240453:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204587.240461:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204587.240463:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204587.240470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204587.240473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204587.240478:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204587.240480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204587.240483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204587.240486:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204587.240502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204587.240508:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204587.240510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204587.240513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204587.240516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204587.240519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204587.240521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204587.240523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204587.240525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204587.432473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204587.432480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204587.432485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204587.432486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204587.432489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204587.432491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204587.432493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204587.432494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204587.432565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204587.432571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204587.432574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204588.200493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204588.200500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204588.200502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204588.200507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204588.200510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204588.200514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204588.200516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204588.200518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204588.200520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204588.264503:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204588.264512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204588.264515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204588.264522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204588.264527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204588.264531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204588.264535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204588.264538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204588.264541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204588.264660:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204588.264663:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204588.264665:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204588.264667:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204588.264669:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204588.264671:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204588.264673:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204588.264675:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204588.264677:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204588.456508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204588.456516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204588.456523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204588.456524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204588.456528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204588.456529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204588.456533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204588.456534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204588.456555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204588.456558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204588.456562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204589.224476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204589.224484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204589.224486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204589.224490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204589.224494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204589.224497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204589.224500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204589.224502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204589.224504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204589.288436:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204589.288442:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204589.288444:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204589.288449:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204589.288453:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204589.288456:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204589.288460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204589.288462:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204589.288464:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204589.288483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204589.288488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204589.288489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204589.288493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204589.288495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204589.288497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204589.288500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204589.288501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204589.288503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204589.480502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204589.480514:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204589.480517:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204589.480521:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480525:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204589.480528:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480530:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480533:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204589.480538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204589.480542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204589.480544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204589.480596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204589.480605:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204589.480608:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204590.248494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204590.248504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204590.248507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204590.248514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204590.248519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204590.248528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204590.248532:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204590.248535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204590.248538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.312480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204590.312481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204590.312489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204590.312489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204590.312492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.312497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204590.312500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204590.312501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204590.312506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204590.312506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204590.312512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204590.312513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204590.312514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204590.312518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204590.312521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.312523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204590.312526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204590.312531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204590.504496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204590.504505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204590.504513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204590.504514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204590.504516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204590.504517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204590.504520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204590.504521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204590.504537:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204590.504539:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204590.504542:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204590.824472:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300717569, at 4300718080 00000100:00000040:1.0:1788204590.824472:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300717569, at 4300718080 00000100:00000010:0.0:1788204590.824494:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000010:1.0:1788204590.824496:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00000001:1.0:1788204591.272532:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204591.272543:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204591.272546:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204591.272552:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204591.272557:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204591.272562:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204591.272565:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204591.272568:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204591.272572:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204591.336475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204591.336475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204591.336482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204591.336483:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204591.336486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.336487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204591.336494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204591.336494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204591.336501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204591.336502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204591.336507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.336507:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.336514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.336517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204591.336520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.336520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204591.336524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204591.336528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.336704:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204591.336707:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204591.336709:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204591.336718:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.336729:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.336731:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.336736:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204591.336739:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204591.336744:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204591.336751:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205877888:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204591.336755:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204591.336758:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204591.336762:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.336767:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204591.336773:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204591.336778:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204591.336790:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204591.336797:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204591.336809:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204591.336820:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205877888, portal 25 00000100:00000001:2.0:1788204591.336827:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204591.336831:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300828672 : -113511408722944 : ffff98c310ce1c00) 00000100:00000040:2.0:1788204591.336841:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.336856:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204591.336862:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204591.336867:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204591.336874:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205877888, offset 0 00000400:00000200:2.0:1788204591.336887:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204591.336899:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.336908:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204591.336916:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204591.336921:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204591.336924:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204591.336926:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204591.336929:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58714:58714:256:4294967295] 192.168.201.151@tcp LPNI seq info [58694:58694:8:4294967295] 00000400:00000200:2.0:1788204591.336932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.336951:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204591.336961:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.336969:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204591.336982:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204591.336989:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204591.337001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204591.337009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204591.337020:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337025:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204591.337034:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204591.337045:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204591.337053:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204591.337057:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204591.337064:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204591.337070:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337096:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337116:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337120:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204591.337128:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204591.337136:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204591.337142:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300722689 00000100:00000010:2.0:1788204591.337150:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c15a0. 00000100:00000001:2.0:1788204591.337159:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204591.337167:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300722689 00000100:00000010:2.0:1788204591.337172:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a20. 00000100:00000001:2.0:1788204591.337177:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204591.337184:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204591.337189:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337196:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204591.337205:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204591/real 1788204591] req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204591.337224:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204591.337228:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204591.337231:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204591.337236:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204591.337239:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204591.337243:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204591.337248:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204591.337258:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337278:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204591.337280:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337283:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337286:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204591.337299:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337305:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204591.337309:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337317:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204591.337319:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337320:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337321:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337322:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204591.337323:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204591.337325:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204591.337328:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204591.337329:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204591.337332:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204591.337336:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205877888:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204591.337339:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337340:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337342:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1c00 x1875062205877888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337346:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204591.337347:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204591.337349:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204591.337352:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204591.337353:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204591.337354:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204591.337355:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204591.337357:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204591.337361:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204591.337362:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204591.337378:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1c00. 00000100:00000001:2.0:1788204591.337380:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204591.337381:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337383:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337390:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204591.337390:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337392:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204591.337393:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204591.337394:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204591.337396:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337400:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.337401:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337403:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204591.337404:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204591.337405:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204591.337407:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878016:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204591.337409:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204591.337410:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204591.337411:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337413:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204591.337415:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204591.337416:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204591.337419:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204591.337421:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204591.337423:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204591.337425:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878016, portal 10 00000100:00000001:2.0:1788204591.337426:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204591.337427:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:2.0:1788204591.337432:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337436:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204591.337437:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204591.337439:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204591.337441:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205878016, offset 0 00000400:00000200:2.0:1788204591.337445:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337449:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337452:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204591.337456:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204591.337458:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204591.337461:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204591.337463:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337466:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58715:58715:256:4294967295] 192.168.201.151@tcp LPNI seq info [58695:58695:8:4294967295] 00000400:00000200:2.0:1788204591.337470:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337476:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204591.337480:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337483:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204591.337487:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204591.337490:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204591.337493:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204591.337509:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204591.337511:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337513:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204591.337515:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204591.337517:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204591.337519:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204591.337521:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204591.337523:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204591.337525:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337530:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337532:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337536:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337537:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204591.337539:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204591.337542:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204591.337544:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204591.337545:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300718747 00000100:00000001:2.0:1788204591.337548:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337549:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300718747 00000100:00000001:2.0:1788204591.337551:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337552:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204591.337553:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337554:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204591.337556:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204591/real 1788204591] req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204591.337561:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204591.337561:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204591.337562:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204591.337563:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204591.337564:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204591.337565:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204591.337567:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204591.337571:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337575:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204591.337576:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337577:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337578:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204591.337580:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337583:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204591.337585:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337589:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204591.337590:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337605:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337606:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337608:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204591.337609:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204591.337611:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204591.337613:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204591.337615:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204591.337618:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204591.337622:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878016:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204591.337624:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204591.337626:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337630:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.337632:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337633:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204591.337634:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204591.337635:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204591.337637:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878144:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204591.337639:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204591.337640:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204591.337641:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337642:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204591.337645:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204591.337646:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204591.337648:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204591.337650:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204591.337652:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204591.337654:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878144, portal 4 00000100:00000001:2.0:1788204591.337655:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204591.337657:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:2.0:1788204591.337661:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337664:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204591.337666:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204591.337667:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204591.337669:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205878144, offset 0 00000400:00000200:2.0:1788204591.337671:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337674:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337676:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204591.337678:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204591.337680:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204591.337681:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204591.337683:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337685:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58716:58716:256:4294967295] 192.168.201.151@tcp LPNI seq info [58696:58696:8:4294967295] 00000400:00000200:2.0:1788204591.337688:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337692:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204591.337694:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337696:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204591.337698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204591.337700:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204591.337703:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204591.337704:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204591.337707:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337708:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204591.337710:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204591.337712:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204591.337714:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204591.337715:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204591.337716:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204591.337718:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337723:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337725:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337729:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337730:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204591.337732:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204591.337734:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204591.337735:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204591.337736:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300718633 00000100:00000001:2.0:1788204591.337738:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337739:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300718633 00000100:00000001:2.0:1788204591.337741:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337742:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204591.337743:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337744:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204591.337746:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204591/real 1788204591] req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204591.337750:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204591.337750:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204591.337751:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204591.337752:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204591.337753:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204591.337754:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204591.337756:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204591.337760:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337764:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204591.337764:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337765:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337766:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204591.337769:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337773:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204591.337775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337779:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204591.337780:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337781:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337782:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337783:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204591.337784:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204591.337785:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204591.337787:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204591.337788:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204591.337790:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204591.337794:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878144:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204591.337795:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204591.337797:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337801:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204591.337802:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337803:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204591.337804:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204591.337806:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204591.337807:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204591.337809:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204591.337810:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204591.337811:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204591.337812:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204591.337814:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204591.337815:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204591.337817:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204591.337818:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204591.337820:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204591.337822:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878272, portal 4 00000100:00000001:2.0:1788204591.337823:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204591.337824:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:2.0:1788204591.337827:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337831:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204591.337832:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204591.337833:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204591.337834:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205878272, offset 0 00000400:00000200:2.0:1788204591.337836:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337839:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204591.337841:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204591.337843:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204591.337844:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204591.337845:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204591.337847:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58717:58717:256:4294967295] 192.168.201.151@tcp LPNI seq info [58697:58697:8:4294967295] 00000400:00000200:2.0:1788204591.337849:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204591.337853:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204591.337855:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337856:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204591.337858:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204591.337860:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204591.337863:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204591.337865:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204591.337868:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204591.337871:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204591.337873:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204591.337877:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204591.337879:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204591.337881:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204591.337883:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204591.337886:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337893:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337895:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337901:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337903:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204591.337905:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204591.337908:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204591.337911:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204591.337913:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300718639 00000100:00000001:2.0:1788204591.337916:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337918:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300718639 00000100:00000001:2.0:1788204591.337920:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300722689 00000100:00000001:2.0:1788204591.337922:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204591.337923:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.337925:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204591.337929:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204591/real 1788204591] req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204591.337936:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204591.337938:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204591.337939:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204591.337941:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204591.337942:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204591.337944:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204591.337946:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204591.337951:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337955:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204591.337955:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337956:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337957:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204591.337959:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337963:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204591.337965:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337969:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204591.337970:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337971:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204591.337972:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337973:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204591.337974:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204591.337975:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204591.337977:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204591.337978:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204591.337980:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204591.337984:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204591.337986:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.337987:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.337990:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205878016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.337994:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204591.337995:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204591.337996:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204591.337997:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204591.337998:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204591.337999:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204591.338000:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204591.338002:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204591.338003:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204591.338004:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204591.338005:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:2.0:1788204591.338007:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204591.338008:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.338009:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.338011:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205878144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.338015:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204591.338015:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204591.338016:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204591.338017:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204591.338018:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204591.338019:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204591.338020:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204591.338020:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204591.338021:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204591.338022:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204591.338022:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:2.0:1788204591.338024:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204591.338024:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.338025:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204591.338026:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205878272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204591.338030:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204591.338031:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204591.338032:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204591.338033:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204591.338034:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204591.338034:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204591.338036:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204591.338036:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204591.338038:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204591.338038:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204591.338039:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:2.0:1788204591.338040:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204591.338041:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.338042:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204591.338044:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204591.338045:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.338046:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204591.338047:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204591.528503:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204591.528506:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204591.528509:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528515:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204591.528517:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528518:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528520:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204591.528524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204591.528527:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204591.528528:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204591.528529:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528530:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204591.528530:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528532:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528533:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204591.528537:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204591.528538:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204591.528539:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528540:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204591.528541:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528542:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204591.528543:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204591.528612:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204591.528615:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204591.528619:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204592.296481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204592.296489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204592.296490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204592.296495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204592.296498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204592.296503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204592.296505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204592.296507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204592.296509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204592.360456:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204592.360463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204592.360466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204592.360467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204592.360470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204592.360472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204592.360478:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204592.360479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204592.360483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204592.360484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204592.360488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204592.360489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204592.360493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204592.360496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204592.360498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204592.360500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204592.360502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204592.360505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204592.552506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204592.552513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204592.552518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204592.552519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204592.552522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204592.552523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204592.552525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204592.552526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204592.552540:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204592.552542:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204592.552545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204593.320502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204593.320510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204593.320512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204593.320517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204593.320520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204593.320523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204593.320525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204593.320527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204593.320529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204593.384492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204593.384492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204593.384500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204593.384501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204593.384503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204593.384505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204593.384510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204593.384511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204593.384515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204593.384516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204593.384519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204593.384521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204593.384522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204593.384525:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204593.384528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204593.384532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204593.384536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204593.384538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204593.576441:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204593.576448:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204593.576453:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204593.576453:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204593.576456:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204593.576457:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204593.576459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204593.576460:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204593.576514:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204593.576517:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204593.576520:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204594.344473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204594.344482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204594.344484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204594.344490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204594.344495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204594.344500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204594.344503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204594.344506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204594.344510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204594.408447:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204594.408447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204594.408456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204594.408458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204594.408461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204594.408467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204594.408468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204594.408474:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204594.408475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204594.408480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204594.408481:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204594.408483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204594.408486:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204594.408491:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204594.408495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204594.408496:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204594.408499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204594.408504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204594.600507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204594.600513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204594.600518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204594.600519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204594.600521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204594.600522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204594.600524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204594.600525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204594.600550:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204594.600553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204594.600555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204595.368542:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204595.368550:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204595.368552:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204595.368559:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204595.368563:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204595.368567:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204595.368569:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204595.368572:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204595.368575:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204595.432492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204595.432492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204595.432502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204595.432502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204595.432506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204595.432507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204595.432515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204595.432516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204595.432533:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204595.432538:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204595.432540:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204595.432543:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204595.432546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204595.432547:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204595.432551:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204595.432554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204595.432556:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204595.432558:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204595.624492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204595.624499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204595.624507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204595.624508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204595.624511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204595.624512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204595.624515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204595.624516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204595.624601:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204595.624606:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204595.624610:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204595.944464:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300722689, at 4300723200 00000100:00000040:0.0:1788204595.944467:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300722689, at 4300723200 00000100:00000010:1.0:1788204595.944483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000010:0.0:1788204595.944487:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00000001:1.0:1788204596.392488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204596.392497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204596.392499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204596.392519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204596.392523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204596.392528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204596.392531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204596.392534:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204596.392537:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204596.456446:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204596.456446:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204596.456456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204596.456456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204596.456458:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.456459:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204596.456467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204596.456467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204596.456474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204596.456475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204596.456478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204596.456481:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.456484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204596.456491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204596.456494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.456495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.456501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.456515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.456742:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204596.456750:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204596.456755:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204596.456769:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.456788:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.456792:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.456801:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204596.456805:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204596.456808:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204596.456821:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205878400:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204596.456836:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204596.456850:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204596.456853:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.456861:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204596.456868:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204596.456872:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204596.456883:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204596.456893:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204596.456903:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204596.456915:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878400, portal 25 00000100:00000001:2.0:1788204596.456922:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204596.456924:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000040:2.0:1788204596.456938:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.456959:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204596.456966:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204596.456972:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204596.456981:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205878400, offset 0 00000400:00000200:2.0:1788204596.456998:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204596.457009:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.457021:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204596.457031:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204596.457039:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204596.457044:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204596.457048:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204596.457057:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58718:58718:256:4294967295] 192.168.201.151@tcp LPNI seq info [58698:58698:8:4294967295] 00000400:00000200:2.0:1788204596.457065:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.457085:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204596.457093:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.457097:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204596.457107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204596.457112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204596.457135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204596.457144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204596.457159:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.457165:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204596.457176:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204596.457195:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204596.457204:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204596.457214:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204596.457224:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204596.457237:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457265:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.457273:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457286:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.457289:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204596.457298:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204596.457306:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204596.457311:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300727809 00000100:00000010:2.0:1788204596.457320:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c17e0. 00000100:00000001:2.0:1788204596.457330:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204596.457342:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300727809 00000100:00000010:2.0:1788204596.457348:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1cc0. 00000100:00000001:2.0:1788204596.457352:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204596.457355:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204596.457359:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.457362:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204596.457398:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204596/real 1788204596] req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204596.457441:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204596.457448:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204596.457450:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204596.457454:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204596.457456:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204596.457459:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204596.457464:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204596.457516:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457542:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204596.457544:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457549:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.457557:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204596.457570:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457604:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204596.457615:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457649:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204596.457659:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457671:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.457676:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457685:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204596.457692:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204596.457705:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204596.457717:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204596.457724:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204596.457739:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204596.457776:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205878400:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204596.457790:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457797:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.457807:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205878400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457834:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204596.457836:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204596.457838:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204596.457842:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204596.457845:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204596.457847:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204596.457849:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204596.457852:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204596.457855:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204596.457857:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204596.457858:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:2.0:1788204596.457860:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204596.457861:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457863:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.457869:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204596.457870:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.457872:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204596.457873:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204596.457874:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204596.457877:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457882:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.457883:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.457884:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204596.457886:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204596.457887:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204596.457889:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878528:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204596.457891:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204596.457893:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204596.457894:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.457895:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204596.457897:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204596.457899:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204596.457902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204596.457905:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204596.457907:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204596.457910:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878528, portal 10 00000100:00000001:2.0:1788204596.457911:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204596.457912:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:2.0:1788204596.457917:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457921:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204596.457922:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204596.457924:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204596.457925:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205878528, offset 0 00000400:00000200:2.0:1788204596.457927:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204596.457930:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.457932:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204596.457934:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204596.457936:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204596.457938:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204596.457939:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204596.457941:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58719:58719:256:4294967295] 192.168.201.151@tcp LPNI seq info [58699:58699:8:4294967295] 00000400:00000200:2.0:1788204596.457943:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.457948:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204596.457951:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.457953:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204596.457955:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204596.457957:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204596.457960:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204596.457962:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204596.457965:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.457966:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204596.457968:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204596.457970:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204596.457972:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204596.457973:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204596.457975:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204596.457977:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457982:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.457984:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.457988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.457989:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204596.457991:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204596.457993:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204596.457995:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204596.457996:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300723867 00000100:00000001:2.0:1788204596.457998:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.457999:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300723867 00000100:00000001:2.0:1788204596.458001:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.458002:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204596.458003:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458004:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204596.458006:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204596/real 1788204596] req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204596.458010:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204596.458011:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204596.458012:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204596.458013:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204596.458014:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204596.458015:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204596.458017:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204596.458021:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458029:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204596.458030:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458031:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458032:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204596.458034:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458037:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204596.458039:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458043:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204596.458044:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458045:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458046:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458047:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204596.458048:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204596.458049:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204596.458058:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204596.458060:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204596.458064:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204596.458069:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878528:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204596.458071:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204596.458073:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458077:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.458078:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.458079:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204596.458081:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204596.458082:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204596.458084:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878656:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204596.458086:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204596.458087:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204596.458088:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.458088:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204596.458090:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204596.458092:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204596.458094:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204596.458095:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204596.458097:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204596.458099:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878656, portal 4 00000100:00000001:2.0:1788204596.458100:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204596.458101:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:2.0:1788204596.458105:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458109:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204596.458110:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204596.458112:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204596.458113:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205878656, offset 0 00000400:00000200:2.0:1788204596.458115:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204596.458117:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.458119:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204596.458120:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204596.458122:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204596.458124:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204596.458126:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204596.458129:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58720:58720:256:4294967295] 192.168.201.151@tcp LPNI seq info [58700:58700:8:4294967295] 00000400:00000200:2.0:1788204596.458133:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.458138:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204596.458143:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.458146:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204596.458150:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204596.458153:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204596.458157:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204596.458160:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204596.458164:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.458166:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204596.458168:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204596.458171:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204596.458173:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204596.458174:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204596.458176:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204596.458179:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458183:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.458185:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458189:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458190:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204596.458192:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204596.458194:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204596.458196:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204596.458198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300723753 00000100:00000001:2.0:1788204596.458199:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.458201:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300723753 00000100:00000001:2.0:1788204596.458202:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.458203:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204596.458204:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458205:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204596.458207:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204596/real 1788204596] req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204596.458212:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204596.458213:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204596.458214:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204596.458216:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204596.458217:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204596.458218:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204596.458220:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204596.458224:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458228:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204596.458228:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458229:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458230:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204596.458232:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458235:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204596.458237:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458241:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204596.458242:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458243:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458244:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458245:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204596.458247:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204596.458248:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204596.458250:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204596.458251:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204596.458254:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204596.458259:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878656:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204596.458261:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204596.458263:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458267:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204596.458267:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.458269:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204596.458270:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204596.458271:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204596.458272:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204596.458274:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204596.458275:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204596.458276:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204596.458277:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204596.458279:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204596.458280:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204596.458282:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204596.458283:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204596.458285:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204596.458287:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878784, portal 4 00000100:00000001:2.0:1788204596.458288:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204596.458289:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:2.0:1788204596.458292:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458296:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204596.458298:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204596.458299:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204596.458300:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205878784, offset 0 00000400:00000200:2.0:1788204596.458303:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204596.458305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.458307:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204596.458309:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204596.458311:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204596.458312:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204596.458314:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204596.458315:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58721:58721:256:4294967295] 192.168.201.151@tcp LPNI seq info [58701:58701:8:4294967295] 00000400:00000200:2.0:1788204596.458318:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204596.458322:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204596.458324:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.458326:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204596.458328:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204596.458329:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204596.458332:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204596.458334:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204596.458336:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204596.458337:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204596.458339:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204596.458341:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204596.458343:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204596.458344:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204596.458345:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204596.458347:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458351:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.458352:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458356:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458357:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204596.458358:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204596.458360:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204596.458362:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204596.458379:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300723760 00000100:00000001:2.0:1788204596.458381:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.458382:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300723760 00000100:00000001:2.0:1788204596.458383:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300727809 00000100:00000001:2.0:1788204596.458385:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204596.458386:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458387:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204596.458389:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204596/real 1788204596] req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204596.458399:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204596.458400:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204596.458401:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204596.458403:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204596.458404:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204596.458405:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204596.458409:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204596.458414:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458418:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204596.458419:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458420:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458421:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204596.458423:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458426:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204596.458428:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458432:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204596.458434:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458435:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204596.458436:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458437:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204596.458439:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204596.458440:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204596.458442:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204596.458443:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204596.458445:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204596.458449:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205878784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204596.458451:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458452:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.458454:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205878528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458458:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204596.458459:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204596.458460:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204596.458462:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204596.458463:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204596.458463:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204596.458465:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204596.458466:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:2.0:1788204596.458468:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204596.458468:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204596.458469:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204596.458471:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204596.458472:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458473:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.458475:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205878656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458486:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204596.458488:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204596.458489:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204596.458491:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204596.458492:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204596.458494:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204596.458496:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204596.458497:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204596.458499:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204596.458500:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204596.458502:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:2.0:1788204596.458514:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204596.458515:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458517:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204596.458521:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205878784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204596.458528:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204596.458529:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204596.458530:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204596.458533:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204596.458535:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204596.458536:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204596.458537:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204596.458538:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204596.458540:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204596.458541:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204596.458541:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:2.0:1788204596.458543:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204596.458543:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458545:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.458547:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204596.458548:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.458549:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204596.458550:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204596.648555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204596.648562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204596.648566:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204596.648567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204596.648569:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204596.648570:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204596.648572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204596.648573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204596.648591:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204596.648593:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204596.648595:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204597.416520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204597.416529:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204597.416531:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204597.416538:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204597.416542:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204597.416547:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204597.416551:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204597.416553:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204597.416557:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204597.480502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204597.480502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204597.480512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204597.480512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204597.480515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.480515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204597.480521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204597.480521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204597.480525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204597.480525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204597.480530:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204597.480530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204597.480533:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204597.480537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204597.480540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.480540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204597.480542:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204597.480543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204597.672526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204597.672534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204597.672542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204597.672544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204597.672548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204597.672549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204597.672555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204597.672556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204597.672601:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204597.672610:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204597.672617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204598.440517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204598.440524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204598.440527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204598.440532:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204598.440535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204598.440539:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204598.440541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204598.440543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204598.440545:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204598.504509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204598.504509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204598.504520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204598.504520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204598.504522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204598.504524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204598.504530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204598.504531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204598.504536:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204598.504537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204598.504540:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204598.504541:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204598.504543:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204598.504546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204598.504549:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204598.504556:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204598.504559:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204598.504563:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204598.696554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204598.696581:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204598.696588:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204598.696590:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204598.696597:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204598.696598:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204598.696602:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204598.696603:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204598.696625:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204598.696629:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204598.696632:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204599.464444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204599.464469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204599.464471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204599.464476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204599.464479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204599.464483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204599.464485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204599.464487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204599.464489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204599.528533:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204599.528543:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204599.528546:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204599.528552:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204599.528555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204599.528559:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204599.528561:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204599.528564:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204599.528566:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204599.528674:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204599.528677:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204599.528678:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204599.528681:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204599.528683:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204599.528685:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204599.528688:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204599.528689:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204599.528692:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204599.720528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204599.720543:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204599.720547:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204599.720551:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720555:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204599.720558:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720560:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720562:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720566:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204599.720567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204599.720571:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204599.720572:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204599.720644:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204599.720651:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204599.720656:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204600.488483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204600.488490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204600.488492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204600.488497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204600.488500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204600.488503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204600.488506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204600.488508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204600.488510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204600.552459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204600.552465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204600.552467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204600.552471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204600.552474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204600.552478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204600.552481:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204600.552484:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204600.552486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204600.552488:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204600.552495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204600.552499:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204600.552504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204600.552509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204600.552513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204600.552516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204600.552519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204600.552522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204600.744482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204600.744488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204600.744494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204600.744495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204600.744498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204600.744499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204600.744502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204600.744502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204600.744515:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204600.744518:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204600.744520:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204601.064464:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300727809, at 4300728320 00000100:00000040:1.0:1788204601.064464:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300727809, at 4300728320 00000100:00000010:0.0:1788204601.064476:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000010:1.0:1788204601.064479:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000001:1.0:1788204601.512464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204601.512470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204601.512471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204601.512475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204601.512478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204601.512481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204601.512482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204601.512484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204601.512486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204601.576466:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204601.576477:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204601.576479:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204601.576483:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204601.576488:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204601.576496:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6056 00000400:00000200:1.0:1788204601.576508:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204601.576514:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204601.576517:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204601.576522:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576528:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204601.576528:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204601.576538:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1788204601.576541:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204601.576543:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576550:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:2.0:1788204601.576551:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204601.576559:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788204601.576561:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204601.576564:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788204601.576566:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204601.576573:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204601.576577:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:2.0:1788204601.576578:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204601.576584:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788204601.576589:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788204601.576591:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204601.576595:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204601.576600:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576603:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.576604:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576606:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204601.576608:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204601.576611:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576613:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204601.576616:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204601.576618:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204601.576620:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576622:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204601.576623:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204601.576631:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788204601.576633:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204601.576636:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204601.576638:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576640:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204601.576643:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204601.576647:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204601.576653:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:1.0:1788204601.576660:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204601.576668:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204601.576678:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204601.576682:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204601.576686:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204601.576706:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204601.576709:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204601.576712:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204601.576713:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204601.576714:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204601.576716:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204601.576718:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6056 00000400:00000200:1.0:1788204601.576722:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204601.576725:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204601.576726:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204601.576729:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576730:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.576732:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576733:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204601.576735:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204601.576735:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1788204601.576742:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204601.576744:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204601.576744:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204601.576750:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788204601.576751:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204601.576758:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204601.576760:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204601.576772:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:2.0:1788204601.576773:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204601.576778:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.576786:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576795:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204601.576803:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204601.576813:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788204601.576816:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204601.576818:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576820:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204601.576823:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204601.576825:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204601.576828:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204601.576832:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.576832:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204601.576839:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204601.576844:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204601.576844:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788204601.576854:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576859:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204601.576860:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205878912:192.168.201.151@tcp:250:kworker.0 00000100:00000010:1.0:1788204601.576865:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000020:00000040:1.0:1788204601.576868:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204601.576870:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204601.576871:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.576873:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204601.576876:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204601.576878:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204601.576888:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:2.0:1788204601.576888:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788204601.576895:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.576897:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1788204601.576903:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204601.576904:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.576914:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788204601.576916:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204601.576919:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204601.576921:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204601.576925:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:1.0:1788204601.576928:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.576931:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204601.576932:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204601.576937:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204601.576938:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204601.576939:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204601.576941:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204601.576947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788204601.576948:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6056 00000400:00000200:1.0:1788204601.576961:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204601.576963:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:1.0:1788204601.576965:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204601.576970:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204601.576973:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576975:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1788204601.576976:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000040:1.0:1788204601.576978:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.576982:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204601.576984:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204601.576984:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204601.576985:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204601.576987:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000200:2.0:1788204601.576989:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205878912, portal 25 00000020:00000001:1.0:1788204601.576990:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:2.0:1788204601.577000:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204601.577003:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000020:00000001:1.0:1788204601.577005:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204601.577019:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:2.0:1788204601.577019:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204601.577029:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.577033:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.577042:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204601.577049:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204601.577057:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204601.577060:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204601.577061:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204601.577067:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204601.577070:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204601.577074:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788204601.577075:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.577079:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:2.0:1788204601.577080:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000008:00100000:1.0:1788204601.577081:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204601.577088:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.577091:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204601.577091:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205878912, offset 0 00000100:00000010:1.0:1788204601.577099:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204601.577101:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204601.577105:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204601.577108:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577111:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204601.577112:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204601.577121:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204601.577122:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204601.577125:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788204601.577127:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577134:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:1.0:1788204601.577135:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204601.577147:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204601.577149:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577154:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:1.0:1788204601.577158:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204601.577168:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204601.577169:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204601.577175:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204601.577177:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204601.577181:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1788204601.577182:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:1.0:1788204601.577184:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204601.577187:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204601.577196:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6056 00000400:00000200:2.0:1788204601.577196:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204601.577210:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788204601.577211:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204601.577230:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204601.577237:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58722:58722:256:4294967295] 192.168.201.151@tcp LPNI seq info [58702:58702:8:4294967295] 00000100:00080000:1.0:1788204601.577238:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204601.577249:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.577257:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.577266:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204601.577266:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204601.577275:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204601.577277:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204601.577278:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204601.577280:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204601.577281:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204601.577286:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204601.577293:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204601.577307:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204601.577308:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788204601.577309:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204601.577317:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204601.577324:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204601.577328:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204601.577335:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:2.0:1788204601.577335:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204601.577342:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204601.577345:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204601.577350:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1788204601.577350:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 02000000:00000001:1.0:1788204601.577359:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.577361:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204601.577362:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204601.577385:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204601.577386:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204601.577389:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788204601.577391:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204601.577393:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204601.577397:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204601.577401:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204601.577402:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204601.577404:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204601.577408:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204601.577412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 02000000:00000010:1.0:1788204601.577413:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788204601.577432:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204601.577442:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1788204601.577444:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204601.577457:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1788204601.577457:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1788204601.577462:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204601.577478:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1788204601.577479:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.577487:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000100:00000001:1.0:1788204601.577500:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577500:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1788204601.577506:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6066) 00000400:00000200:2.0:1788204601.577510:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204601.577518:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204601.577522:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204601.577529:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204601.577534:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577545:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.577549:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577557:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.577559:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204601.577566:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577570:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204601.577574:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300732930 00000100:00000010:2.0:1788204601.577580:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1420. 00000100:00000001:2.0:1788204601.577588:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204601.577602:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300732930 00000100:00000010:2.0:1788204601.577605:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c60. 00000100:00000001:2.0:1788204601.577609:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204601.577612:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204601.577616:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.577618:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204601.577625:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204601/real 1788204601] req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204601.577636:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204601.577638:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204601.577640:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204601.577643:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204601.577645:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204601.577648:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204601.577652:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204601.577661:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577669:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204601.577671:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577674:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.577676:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204601.577680:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577688:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204601.577693:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577707:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204601.577710:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577712:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.577713:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577716:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204601.577719:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204601.577722:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204601.577726:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204601.577729:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204601.577734:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204601.577743:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205878912:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204601.577748:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577750:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.577753:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205878912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577761:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204601.577764:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204601.577767:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204601.577770:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204601.577773:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204601.577775:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204601.577777:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204601.577780:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:2.0:1788204601.577784:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204601.577786:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204601.577789:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:2.0:1788204601.577792:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204601.577794:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577796:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.577805:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204601.577806:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.577809:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204601.577811:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204601.577812:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204601.577816:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577824:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204601.577826:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.577828:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204601.577830:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204601.577833:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204601.577836:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879040:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204601.577839:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204601.577841:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204601.577843:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.577845:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204601.577847:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204601.577850:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.577854:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204601.577857:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204601.577860:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204601.577863:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879040, portal 10 00000100:00000001:2.0:1788204601.577865:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204601.577867:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:2.0:1788204601.577874:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577881:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204601.577884:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204601.577886:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204601.577888:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205879040, offset 0 00000400:00000200:2.0:1788204601.577892:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204601.577896:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.577900:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204601.577904:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204601.577906:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204601.577909:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204601.577911:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204601.577915:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58723:58723:256:4294967295] 192.168.201.151@tcp LPNI seq info [58703:58703:8:4294967295] 00000400:00000200:2.0:1788204601.577919:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.577926:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204601.577931:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.577934:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204601.577938:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204601.577941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204601.577946:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204601.577949:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204601.577953:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.577955:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204601.577958:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204601.577961:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204601.577964:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204601.577966:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204601.577969:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204601.577972:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577980:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.577983:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.577990:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.577992:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204601.577995:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204601.577999:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204601.578002:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204601.578004:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300728987 00000100:00000001:2.0:1788204601.578007:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578009:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300728987 00000100:00000001:2.0:1788204601.578012:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578014:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204601.578016:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578018:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204601.578022:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204601/real 1788204601] req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204601.578030:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204601.578032:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204601.578033:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204601.578035:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204601.578037:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204601.578039:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204601.578043:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204601.578051:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578058:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204601.578060:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578061:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578063:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204601.578066:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578073:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204601.578077:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578084:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204601.578087:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578089:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578091:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578093:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204601.578096:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204601.578098:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204601.578101:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204601.578104:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204601.578107:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204601.578116:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879040:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204601.578120:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204601.578124:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578131:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204601.578133:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.578136:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204601.578138:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204601.578140:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204601.578143:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879168:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204601.578146:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204601.578148:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204601.578150:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.578152:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204601.578156:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204601.578158:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.578161:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204601.578164:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204601.578167:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204601.578170:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879168, portal 4 00000100:00000001:2.0:1788204601.578172:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204601.578174:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:2.0:1788204601.578179:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578186:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204601.578189:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204601.578191:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204601.578193:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205879168, offset 0 00000400:00000200:2.0:1788204601.578197:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204601.578201:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.578204:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204601.578207:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204601.578210:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204601.578212:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204601.578214:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204601.578218:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58724:58724:256:4294967295] 192.168.201.151@tcp LPNI seq info [58704:58704:8:4294967295] 00000400:00000200:2.0:1788204601.578222:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.578229:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204601.578233:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.578236:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204601.578240:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204601.578243:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204601.578247:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204601.578250:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204601.578254:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.578256:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204601.578259:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204601.578263:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204601.578265:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204601.578268:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204601.578270:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204601.578274:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578281:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.578285:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578292:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578294:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204601.578296:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204601.578300:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204601.578303:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204601.578306:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300728873 00000100:00000001:2.0:1788204601.578308:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578311:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300728873 00000100:00000001:2.0:1788204601.578313:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578315:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204601.578317:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578319:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204601.578323:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204601/real 1788204601] req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204601.578331:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204601.578333:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204601.578334:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204601.578337:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204601.578339:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204601.578341:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204601.578344:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204601.578353:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578359:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204601.578361:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578363:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578384:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204601.578389:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578397:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204601.578401:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578408:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204601.578412:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578428:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578430:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578433:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204601.578435:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204601.578438:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204601.578443:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204601.578445:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204601.578450:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204601.578458:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879168:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204601.578462:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204601.578466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578473:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204601.578475:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.578477:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204601.578479:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204601.578482:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204601.578485:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204601.578489:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204601.578492:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204601.578494:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204601.578496:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204601.578499:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c800. 02000000:00000001:2.0:1788204601.578501:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204601.578505:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204601.578507:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204601.578510:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204601.578513:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879296, portal 4 00000100:00000001:2.0:1788204601.578515:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204601.578517:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000040:2.0:1788204601.578523:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578529:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204601.578531:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204601.578533:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204601.578536:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205879296, offset 0 00000400:00000200:2.0:1788204601.578540:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204601.578543:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.578547:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204601.578550:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204601.578552:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204601.578555:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204601.578557:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204601.578560:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58725:58725:256:4294967295] 192.168.201.151@tcp LPNI seq info [58705:58705:8:4294967295] 00000400:00000200:2.0:1788204601.578564:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204601.578571:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204601.578576:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.578579:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204601.578583:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204601.578586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204601.578590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204601.578593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204601.578597:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204601.578600:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204601.578603:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204601.578607:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204601.578610:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204601.578611:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204601.578614:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204601.578617:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578625:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.578628:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578634:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578636:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204601.578638:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204601.578642:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204601.578645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204601.578647:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300728880 00000100:00000001:2.0:1788204601.578649:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578651:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300728880 00000100:00000001:2.0:1788204601.578654:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300732930 00000100:00000001:2.0:1788204601.578656:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204601.578658:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578660:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204601.578664:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204601/real 1788204601] req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204601.578672:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204601.578673:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204601.578675:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204601.578678:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204601.578679:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204601.578682:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204601.578685:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204601.578694:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578701:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204601.578702:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578705:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578706:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204601.578710:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578717:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204601.578721:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578728:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204601.578731:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578733:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204601.578734:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578736:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204601.578738:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204601.578741:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204601.578744:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204601.578747:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204601.578751:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204601.578759:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204601.578763:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578765:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.578769:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205879040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578775:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204601.578777:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204601.578779:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204601.578781:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204601.578783:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204601.578784:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204601.578786:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204601.578788:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:2.0:1788204601.578791:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204601.578792:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204601.578794:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:2.0:1788204601.578796:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204601.578798:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578800:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.578803:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205879168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578811:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204601.578812:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204601.578814:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204601.578816:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204601.578817:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204601.578818:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204601.578820:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204601.578822:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:2.0:1788204601.578824:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204601.578826:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204601.578827:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:2.0:1788204601.578830:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204601.578831:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578833:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204601.578836:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205879296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204601.578843:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204601.578845:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204601.578846:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c800. 02000000:00000001:2.0:1788204601.578849:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204601.578851:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204601.578852:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204601.578854:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204601.578856:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:2.0:1788204601.578858:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204601.578859:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204601.578860:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:2.0:1788204601.578862:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204601.578864:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578866:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204601.578869:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204601.578871:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204601.578873:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204601.578875:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768455:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204601.768462:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204601.768464:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204601.768466:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768470:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204601.768473:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768475:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768477:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204601.768481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204601.768484:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204601.768485:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204601.768486:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768486:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204601.768488:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768489:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768490:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204601.768493:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204601.768494:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204601.768495:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768495:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204601.768496:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768497:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204601.768498:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204601.768545:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204601.768548:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204601.768551:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204602.536538:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204602.536544:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204602.536546:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204602.536551:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204602.536555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204602.536560:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204602.536563:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204602.536566:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204602.536569:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204602.600490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204602.600499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204602.600503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204602.600510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204602.600516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204602.600523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204602.600535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204602.600538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204602.600540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204602.792553:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204602.792559:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204602.792564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204602.792566:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204602.792569:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204602.792571:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204602.792574:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204602.792575:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204602.792595:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204602.792598:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204602.792601:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204603.112520:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204603.112527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204603.112529:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204603.112535:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204603.112538:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204603.112542:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204603.112546:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204603.112550:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204603.112553:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204603.560490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204603.560497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204603.560499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204603.560503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204603.560506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204603.560510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204603.560512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204603.560515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204603.560518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.624530:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204603.624537:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204603.624541:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204603.624549:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204603.624557:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204603.624565:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204603.624575:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204603.624579:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204603.624584:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204603.816473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204603.816479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204603.816485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204603.816486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204603.816489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204603.816491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204603.816494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204603.816495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204603.816571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204603.816576:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204603.816580:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204604.136506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204604.136517:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204604.136518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204604.136523:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204604.136526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204604.136530:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.136533:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204604.136536:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.136539:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204604.584433:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204604.584438:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204604.584439:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204604.584442:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204604.584445:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204604.584448:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204604.584449:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204604.584451:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204604.584453:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204604.648492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204604.648503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204604.648508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204604.648516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204604.648522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204604.648528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.648542:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204604.648547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.648551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204604.840531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204604.840537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204604.840542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204604.840543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204604.840546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204604.840547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204604.840550:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204604.840551:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204604.840569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204604.840571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204604.840574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204605.160477:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204605.160484:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204605.160486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204605.160498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204605.160501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204605.160505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204605.160508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204605.160511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204605.160515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204605.608436:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204605.608442:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204605.608444:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204605.608450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204605.608453:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204605.608458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204605.608460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204605.608463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204605.608466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204605.672482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204605.672492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204605.672497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204605.672505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204605.672513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204605.672520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204605.672525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204605.672528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204605.672531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204605.864468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204605.864475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204605.864485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204605.864486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204605.864489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204605.864491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204605.864494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204605.864495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204605.864543:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204605.864546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204605.864548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204606.184496:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300732930, at 4300733440 00000100:00000040:0.0:1788204606.184500:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300732930, at 4300733440 00000100:00000001:2.0:1788204606.184500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204606.184508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204606.184511:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204606.184516:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000010:0.0:1788204606.184518:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00100000:2.0:1788204606.184518:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204606.184527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204606.184533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.184537:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204606.184541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.184545:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204606.632510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204606.632520:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204606.632523:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204606.632529:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204606.632533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204606.632538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204606.632541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204606.632544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204606.632548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.696460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204606.696462:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204606.696470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204606.696473:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204606.696474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696477:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204606.696481:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204606.696481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204606.696488:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204606.696488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204606.696494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204606.696500:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6061 00000100:00000001:2.0:1788204606.696507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204606.696508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.696512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204606.696522:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204606.696530:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204606.696533:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204606.696536:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696539:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.696541:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696542:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204606.696545:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204606.696547:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204606.696550:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204606.696552:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204606.696554:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204606.696555:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204606.696563:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696564:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.696565:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696567:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204606.696570:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204606.696573:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696576:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204606.696580:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204606.696583:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204606.696586:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696589:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204606.696591:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204606.696603:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204606.696607:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204606.696619:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204606.696622:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696625:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204606.696628:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204606.696632:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204606.696642:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204606.696656:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204606.696668:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204606.696679:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204606.696684:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.696690:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204606.696710:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204606.696714:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204606.696717:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204606.696718:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204606.696719:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204606.696721:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204606.696724:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6061 00000400:00000200:0.0:1788204606.696727:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204606.696729:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204606.696731:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204606.696733:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204606.696735:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204606.696736:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.696739:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204606.696739:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204606.696742:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204606.696743:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204606.696745:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204606.696747:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204606.696749:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204606.696751:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204606.696753:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204606.696754:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204606.696755:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204606.696757:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204606.696765:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204606.696766:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204606.696767:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.696770:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696772:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204606.696773:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1788204606.696774:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204606.696776:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788204606.696778:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:2.0:1788204606.696779:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696782:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204606.696786:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788204606.696786:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205879424:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204606.696791:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204606.696793:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204606.696796:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204606.696797:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.696799:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696800:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204606.696804:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788204606.696805:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204606.696809:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696810:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204606.696810:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204606.696815:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204606.696817:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204606.696820:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:2.0:1788204606.696823:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204606.696824:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204606.696826:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.696828:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1788204606.696828:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 02000000:00000001:0.0:1788204606.696831:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204606.696834:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204606.696837:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 02000000:00000010:0.0:1788204606.696838:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204606.696842:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204606.696845:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879424, portal 25 00000100:00100000:0.0:1788204606.696849:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204606.696849:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204606.696852:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204606.696860:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204606.696861:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204606.696864:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.696871:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.696873:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204606.696879:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204606.696883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204606.696886:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205879424, offset 0 00000100:00000001:0.0:1788204606.696889:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204606.696892:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204606.696895:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204606.696896:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204606.696899:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204606.696900:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204606.696903:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204606.696905:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6061 00000400:00000200:0.0:1788204606.696907:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204606.696909:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204606.696911:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204606.696915:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204606.696916:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696921:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.696923:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696925:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204606.696927:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:0.0:1788204606.696930:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204606.696933:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204606.696935:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204606.696936:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1788204606.696939:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204606.696943:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204606.696944:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204606.696950:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204606.696952:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788204606.696954:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204606.696957:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58726:58726:256:4294967295] 192.168.201.151@tcp LPNI seq info [58706:58706:8:4294967295] 00000100:00000040:0.0:1788204606.696963:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204606.696965:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204606.696970:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.696974:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696978:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204606.696981:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:0.0:1788204606.696983:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204606.696990:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:2.0:1788204606.696990:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204606.696997:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204606.696997:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 02000000:00000001:0.0:1788204606.697000:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204606.697003:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204606.697005:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204606.697008:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204606.697009:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204606.697013:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.0:1788204606.697014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000008:00000001:0.0:1788204606.697016:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.697020:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204606.697023:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000800:00000200:2.1:1788204606.697023:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204606.697029:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.1:1788204606.697029:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 02000000:00000001:0.0:1788204606.697033:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:2.0:1788204606.697036:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204606.697037:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.697041:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:2.0:1788204606.697041:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 02000000:00000001:0.0:1788204606.697044:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204606.697047:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204606.697049:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:0.0:1788204606.697050:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204606.697055:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204606.697057:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1788204606.697061:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204606.697064:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204606.697069:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:0.0:1788204606.697070:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204606.697073:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.697077:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697077:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204606.697085:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204606.697086:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204606.697088:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204606.697094:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204606.697096:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204606.697099:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204606.697104:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204606.697107:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6061 00000100:00000001:2.0:1788204606.697108:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1788204606.697112:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788204606.697114:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788204606.697117:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204606.697121:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204606.697126:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.697128:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204606.697129:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.697133:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204606.697134:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204606.697135:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204606.697136:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1788204606.697137:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204606.697140:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204606.697141:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204606.697143:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204606.697145:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204606.697145:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.697150:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204606.697153:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204606.697153:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204606.697158:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204606.697158:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300738049 00000100:00000001:0.0:1788204606.697163:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000010:2.0:1788204606.697165:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1600. 00000020:00000040:0.0:1788204606.697168:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204606.697173:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204606.697174:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204606.697179:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204606.697183:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204606.697183:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300738049 00000100:00000010:2.0:1788204606.697187:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1660. 02000000:00000001:0.0:1788204606.697188:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204606.697190:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788204606.697194:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697195:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204606.697199:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.697200:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204606.697202:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1788204606.697204:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1788204606.697207:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204606/real 1788204606] req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204606.697210:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204606.697215:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204606.697221:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000010:0.0:1788204606.697224:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000400:00000001:2.0:1788204606.697224:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204606.697227:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000020:00000040:0.0:1788204606.697230:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:2.0:1788204606.697230:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204606.697234:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 02000000:00000001:0.0:1788204606.697238:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204606.697238:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1788204606.697242:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204606.697243:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204606.697247:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204606.697250:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1788204606.697253:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204606.697254:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204606.697262:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:2.0:1788204606.697262:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204606.697263:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697266:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697269:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204606.697270:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204606.697272:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204606.697279:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204606.697282:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204606.697292:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204606.697295:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204606.697296:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.697305:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697306:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204606.697312:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697316:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697318:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697322:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204606.697340:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204606.697347:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6071) 00000100:00000001:2.0:1788204606.697351:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204606.697355:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204606.697360:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204606.697383:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204606.697388:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204606.697395:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205879424:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204606.697399:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697401:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.697404:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205879424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697415:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204606.697417:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204606.697420:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204606.697423:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204606.697425:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204606.697426:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204606.697428:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204606.697431:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:2.0:1788204606.697435:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204606.697437:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204606.697438:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204606.697441:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204606.697442:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697445:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697452:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204606.697454:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697456:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204606.697457:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204606.697459:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204606.697462:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697469:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204606.697471:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697473:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204606.697475:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204606.697477:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204606.697480:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879552:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204606.697483:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204606.697485:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204606.697486:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697487:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204606.697511:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204606.697513:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204606.697516:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204606.697518:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204606.697521:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204606.697523:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879552, portal 10 00000100:00000001:2.0:1788204606.697524:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204606.697525:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204606.697531:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697535:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204606.697536:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204606.697538:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204606.697539:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205879552, offset 0 00000400:00000200:2.0:1788204606.697541:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697544:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.697547:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204606.697551:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204606.697553:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204606.697555:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204606.697558:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697561:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58727:58727:256:4294967295] 192.168.201.151@tcp LPNI seq info [58707:58707:8:4294967295] 00000400:00000200:2.0:1788204606.697564:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.697570:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204606.697575:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.697578:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204606.697582:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204606.697585:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204606.697590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204606.697607:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204606.697610:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.697612:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204606.697614:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204606.697616:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204606.697619:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204606.697620:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204606.697622:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204606.697625:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697630:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.697632:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697636:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697637:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204606.697640:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204606.697643:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204606.697645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204606.697646:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300734107 00000100:00000001:2.0:1788204606.697648:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.697650:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300734107 00000100:00000001:2.0:1788204606.697651:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.697653:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204606.697654:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697655:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204606.697657:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204606/real 1788204606] req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204606.697661:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204606.697662:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204606.697663:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204606.697664:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204606.697665:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204606.697667:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204606.697669:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204606.697673:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697681:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204606.697682:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697683:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697684:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204606.697686:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697691:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204606.697695:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697701:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204606.697703:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697705:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697706:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697708:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204606.697711:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204606.697713:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204606.697717:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204606.697719:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204606.697723:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204606.697732:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879552:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204606.697735:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204606.697738:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697742:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204606.697743:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697745:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204606.697746:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204606.697748:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204606.697749:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879680:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204606.697751:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204606.697753:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204606.697753:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697755:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204606.697757:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204606.697758:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204606.697761:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204606.697763:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204606.697765:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204606.697767:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879680, portal 4 00000100:00000001:2.0:1788204606.697768:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204606.697769:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204606.697773:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697777:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204606.697779:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204606.697780:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204606.697782:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205879680, offset 0 00000400:00000200:2.0:1788204606.697784:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697786:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.697788:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204606.697791:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204606.697792:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204606.697794:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204606.697795:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697797:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58728:58728:256:4294967295] 192.168.201.151@tcp LPNI seq info [58708:58708:8:4294967295] 00000400:00000200:2.0:1788204606.697800:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.697804:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204606.697807:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.697809:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204606.697812:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204606.697813:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204606.697816:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204606.697818:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204606.697820:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.697821:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204606.697823:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204606.697826:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204606.697827:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204606.697828:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204606.697830:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204606.697832:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697837:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.697838:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697842:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697843:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204606.697845:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204606.697847:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204606.697849:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204606.697850:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300733993 00000100:00000001:2.0:1788204606.697852:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.697854:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300733993 00000100:00000001:2.0:1788204606.697855:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.697856:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204606.697858:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.697859:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204606.697860:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204606/real 1788204606] req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204606.697866:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204606.697867:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204606.697868:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204606.697869:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204606.697870:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204606.697871:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204606.697872:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204606.697877:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697880:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204606.697881:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697882:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697883:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204606.697885:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697889:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204606.697891:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697895:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204606.697897:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697899:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.697900:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.697902:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204606.697904:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204606.697907:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204606.697910:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204606.697912:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204606.697916:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204606.697924:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879680:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204606.697928:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204606.697931:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697937:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204606.697939:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697941:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204606.697942:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204606.697943:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204606.697945:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204606.697948:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204606.697949:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204606.697950:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204606.697951:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204606.697953:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204606.697955:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204606.697957:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204606.697959:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204606.697960:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204606.697962:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879808, portal 4 00000100:00000001:2.0:1788204606.697964:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204606.697965:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204606.697969:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.697973:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204606.697974:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204606.697976:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204606.697978:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205879808, offset 0 00000400:00000200:2.0:1788204606.697980:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697983:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.697985:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204606.697987:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204606.697988:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204606.697990:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204606.697992:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204606.697993:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58729:58729:256:4294967295] 192.168.201.151@tcp LPNI seq info [58709:58709:8:4294967295] 00000400:00000200:2.0:1788204606.697996:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204606.698000:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204606.698002:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.698004:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204606.698006:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204606.698008:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204606.698010:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204606.698012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204606.698014:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204606.698016:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204606.698017:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204606.698020:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204606.698021:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204606.698022:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204606.698024:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204606.698026:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698030:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.698032:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698036:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.698037:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204606.698039:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204606.698041:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204606.698042:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204606.698044:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300733999 00000100:00000001:2.0:1788204606.698046:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.698047:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300733999 00000100:00000001:2.0:1788204606.698048:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300738049 00000100:00000001:2.0:1788204606.698049:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204606.698050:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.698051:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204606.698053:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204606/real 1788204606] req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204606.698057:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204606.698058:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204606.698060:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204606.698062:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204606.698063:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204606.698065:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204606.698068:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204606.698075:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698081:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204606.698083:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698085:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.698086:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204606.698089:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698097:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204606.698100:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698107:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204606.698109:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698110:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204606.698111:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698112:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204606.698114:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204606.698116:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204606.698118:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204606.698119:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204606.698122:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204606.698126:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205879808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204606.698129:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698130:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.698132:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205879552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698135:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204606.698136:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204606.698138:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204606.698139:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204606.698140:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204606.698141:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204606.698143:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204606.698144:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:2.0:1788204606.698146:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204606.698147:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204606.698148:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204606.698149:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204606.698150:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698152:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.698154:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205879680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698161:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204606.698162:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204606.698163:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204606.698164:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204606.698165:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204606.698165:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204606.698166:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204606.698167:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:2.0:1788204606.698168:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204606.698169:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204606.698169:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204606.698170:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204606.698171:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698172:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204606.698173:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205879808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204606.698177:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204606.698177:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204606.698178:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204606.698179:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204606.698180:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204606.698180:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204606.698181:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204606.698182:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:2.0:1788204606.698183:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204606.698184:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204606.698184:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204606.698185:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204606.698186:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698187:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204606.698189:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204606.698190:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.698191:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204606.698192:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204606.888538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204606.888548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204606.888554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204606.888556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204606.888559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204606.888560:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204606.888564:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204606.888565:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204606.888585:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204606.888588:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204606.888591:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204607.208440:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204607.208450:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204607.208453:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204607.208460:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204607.208473:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204607.208476:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204607.208479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204607.208483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204607.208486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204607.656522:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204607.656531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204607.656533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204607.656540:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204607.656544:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204607.656549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204607.656552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204607.656555:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204607.656559:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204607.720493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204607.720502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204607.720505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204607.720512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204607.720517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204607.720524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204607.720531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204607.720537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204607.720543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204607.912497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204607.912505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204607.912514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204607.912516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204607.912519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204607.912521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204607.912524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204607.912526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204607.912596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204607.912601:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204607.912605:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204608.232498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204608.232506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204608.232508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204608.232512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204608.232516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204608.232520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204608.232523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204608.232525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204608.232527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204608.680469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204608.680477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204608.680479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204608.680485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204608.680488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204608.680492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204608.680494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204608.680496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204608.680498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204608.744487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204608.744495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204608.744498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204608.744500:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204608.744504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204608.744504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204608.744506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204608.744511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204608.744511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204608.744515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204608.744517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204608.744518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204608.744522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204608.744526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204608.744530:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204608.744530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204608.744532:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204608.744535:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204608.936509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204608.936518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204608.936523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204608.936524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204608.936526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204608.936527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204608.936529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204608.936530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204608.936546:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204608.936547:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204608.936550:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204609.256465:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204609.256474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204609.256475:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204609.256478:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204609.256481:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204609.256484:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.256486:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204609.256488:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.256490:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204609.704465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204609.704470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204609.704472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204609.704475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204609.704479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204609.704482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204609.704485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204609.704487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204609.704490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204609.768471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204609.768471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204609.768477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204609.768477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204609.768480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204609.768481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204609.768486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204609.768487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204609.768491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204609.768492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204609.768496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.768496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.768501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204609.768503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.768505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204609.768506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204609.768509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204609.768511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960458:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204609.960465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204609.960473:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204609.960475:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204609.960479:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960482:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204609.960483:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960484:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960486:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204609.960489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204609.960491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204609.960492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204609.960551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204609.960554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204609.960557:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204610.280463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204610.280468:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204610.280470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204610.280474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204610.280477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204610.280479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.280481:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204610.280483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.280485:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204610.728464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204610.728470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204610.728473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204610.728478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204610.728481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204610.728485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204610.728488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204610.728490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204610.728492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204610.792435:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204610.792437:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204610.792441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204610.792442:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204610.792444:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204610.792445:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204610.792450:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204610.792451:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204610.792454:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204610.792456:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204610.792458:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204610.792459:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.792465:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204610.792468:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.792470:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204610.792471:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204610.792474:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204610.792477:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204610.984495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204610.984500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204610.984503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204610.984504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204610.984505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204610.984507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204610.984509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204610.984509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204610.984525:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204610.984526:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204610.984528:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204611.304453:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300738049, at 4300738560 00000100:00000040:1.0:1788204611.304453:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300738049, at 4300738560 00000100:00000001:2.0:1788204611.304453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204611.304459:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204611.304462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204611.304467:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00100000:2.0:1788204611.304468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204611.304469:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:2.0:1788204611.304474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204611.304479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.304482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204611.304485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.304488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204611.752484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204611.752492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204611.752494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204611.752499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204611.752502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204611.752506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204611.752508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204611.752511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204611.752514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.816476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204611.816483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204611.816484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204611.816489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204611.816491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204611.816495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.816497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204611.816498:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204611.816499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.816503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204611.816506:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204611.816510:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204611.816513:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204611.816518:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204611.816526:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6066 00000400:00000200:0.0:1788204611.816535:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204611.816541:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204611.816545:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204611.816549:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816553:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204611.816556:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204611.816565:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204611.816569:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204611.816575:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204611.816580:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204611.816586:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204611.816592:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204611.816608:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816611:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204611.816613:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816616:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204611.816619:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204611.816622:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816625:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204611.816628:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204611.816630:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204611.816632:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816634:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204611.816636:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204611.816645:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204611.816649:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204611.816652:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204611.816654:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816657:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204611.816659:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204611.816663:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204611.816672:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204611.816678:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204611.816687:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204611.816697:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204611.816701:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204611.816706:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.816726:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204611.816729:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204611.816733:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204611.816734:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204611.816736:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204611.816739:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204611.816742:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6066 00000400:00000200:0.0:1788204611.816747:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204611.816750:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204611.816752:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204611.816753:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788204611.816755:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816758:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204611.816758:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788204611.816759:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816763:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204611.816763:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204611.816767:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204611.816768:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204611.816770:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204611.816773:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000040:2.0:1788204611.816773:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204611.816777:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204611.816781:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204611.816786:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816790:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204611.816793:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.816797:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204611.816798:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788204611.816799:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204611.816802:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204611.816802:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816807:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204611.816808:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204611.816809:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204611.816812:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204611.816814:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204611.816815:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816820:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204611.816821:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204611.816823:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204611.816828:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205879936:192.168.201.151@tcp:250:kworker.0 00000008:00000001:0.0:1788204611.816834:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816836:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.816836:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:0.0:1788204611.816841:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 02000000:00000001:2.0:1788204611.816842:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1788204611.816844:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204611.816847:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204611.816849:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204611.816852:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204611.816853:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.816857:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204611.816858:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1788204611.816860:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:0.0:1788204611.816861:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.816865:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204611.816871:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000400:00000200:2.0:1788204611.816874:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204611.816877:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204611.816880:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00100000:0.0:1788204611.816886:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1788204611.816890:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204611.816897:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1788204611.816899:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205879936, portal 25 00000100:00000001:0.0:1788204611.816900:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.816905:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1788204611.816909:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.816909:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204611.816919:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.816926:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.816929:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788204611.816931:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1788204611.816933:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204611.816937:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00080000:0.0:1788204611.816938:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204611.816942:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1788204611.816943:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205879936, offset 0 00000100:00080000:0.0:1788204611.816944:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204611.816950:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204611.816952:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204611.816955:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6066 00000400:00000200:0.0:1788204611.816960:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.816963:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204611.816967:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204611.816970:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204611.816971:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788204611.816975:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204611.816978:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204611.816979:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204611.816983:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:0.0:1788204611.816985:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204611.816989:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788204611.816990:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204611.816994:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788204611.816995:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204611.816998:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:2.0:1788204611.816999:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58730:58730:256:4294967295] 192.168.201.151@tcp LPNI seq info [58710:58710:8:4294967295] 00000020:00000001:0.0:1788204611.817001:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204611.817004:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:0.0:1788204611.817006:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204611.817010:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204611.817014:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204611.817016:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:0.0:1788204611.817019:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.817022:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1788204611.817025:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204611.817026:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.817029:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000010:2.0:1788204611.817030:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000020:00000040:0.0:1788204611.817031:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204611.817036:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204611.817038:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204611.817040:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1788204611.817041:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204611.817043:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1788204611.817045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 02000000:00000001:0.0:1788204611.817046:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.817051:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204611.817054:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.1:1788204611.817054:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00000001:0.0:1788204611.817057:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.817059:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204611.817059:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000100:00000010:0.0:1788204611.817066:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000800:00000100:2.0:1788204611.817066:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.817070:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000020:00000040:0.0:1788204611.817071:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204611.817075:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204611.817076:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204611.817077:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204611.817078:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204611.817081:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204611.817083:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204611.817083:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000010:0.0:1788204611.817087:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000400:00000200:2.0:1788204611.817088:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204611.817092:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:0.0:1788204611.817093:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817097:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1788204611.817100:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1788204611.817103:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.817129:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204611.817129:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204611.817133:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204611.817134:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204611.817139:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817144:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817148:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204611.817151:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817153:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204611.817154:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204611.817159:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1788204611.817159:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204611.817162:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204611.817163:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300743169 00000100:00080000:0.0:1788204611.817164:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204611.817167:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204611.817169:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6066 00000100:00000010:2.0:1788204611.817169:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1900. 00000400:00000200:0.0:1788204611.817175:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204611.817176:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1788204611.817177:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204611.817181:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204611.817184:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204611.817185:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300743169 00000100:00000001:0.0:1788204611.817187:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1788204611.817189:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a80. 00000100:00000040:0.0:1788204611.817191:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204611.817193:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204611.817194:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204611.817198:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204611.817199:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:2.0:1788204611.817200:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788204611.817201:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204611.817203:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204611.817204:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204611.817207:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204611.817207:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788204611.817210:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00100000:2.0:1788204611.817213:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204611/real 1788204611] req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204611.817215:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.817220:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204611.817223:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204611.817227:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204611.817230:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000001:2.0:1788204611.817231:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1788204611.817234:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000001:2.0:1788204611.817235:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204611.817251:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204611.817253:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204611.817254:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204611.817256:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204611.817259:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.817264:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204611.817267:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1788204611.817270:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204611.817271:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204611.817275:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.817278:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204611.817281:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:2.0:1788204611.817283:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:0.0:1788204611.817284:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817285:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204611.817287:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817288:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:0.0:1788204611.817291:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204611.817291:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204611.817295:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204611.817296:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204611.817301:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204611.817303:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204611.817306:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204611.817308:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204611.817310:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204611.817311:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204611.817315:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000040:2.0:1788204611.817315:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.817320:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817325:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:0.0:1788204611.817328:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204611.817329:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817333:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.817335:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817339:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204611.817342:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204611.817342:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204611.817346:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204611.817347:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1788204611.817352:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204611.817353:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204611.817358:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204611.817384:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204611.817386:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204611.817392:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6076) 00000100:00100000:2.0:1788204611.817400:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205879936:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204611.817405:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817408:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.817411:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205879936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817425:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204611.817428:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204611.817431:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204611.817435:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204611.817436:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204611.817438:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204611.817441:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204611.817443:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:2.0:1788204611.817447:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204611.817449:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204611.817451:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204611.817455:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204611.817456:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817459:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817465:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204611.817467:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817468:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204611.817470:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204611.817471:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204611.817475:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817482:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204611.817483:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817485:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204611.817486:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204611.817489:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204611.817491:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880064:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204611.817495:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204611.817496:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204611.817498:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817500:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204611.817503:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204611.817504:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204611.817508:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204611.817510:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204611.817514:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204611.817517:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880064, portal 10 00000100:00000001:2.0:1788204611.817518:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204611.817520:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204611.817531:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817537:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204611.817540:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204611.817543:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204611.817545:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205880064, offset 0 00000400:00000200:2.0:1788204611.817549:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204611.817553:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.817557:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204611.817560:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204611.817562:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204611.817564:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204611.817566:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204611.817569:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58731:58731:256:4294967295] 192.168.201.151@tcp LPNI seq info [58711:58711:8:4294967295] 00000400:00000200:2.0:1788204611.817573:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.817579:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204611.817583:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.817587:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204611.817590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204611.817593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204611.817597:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204611.817600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1788204611.817603:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.817605:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204611.817607:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204611.817611:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204611.817614:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204611.817615:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204611.817618:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204611.817622:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817629:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.817632:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817639:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817641:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204611.817644:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204611.817647:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204611.817649:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204611.817651:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300739227 00000100:00000001:2.0:1788204611.817652:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.817654:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300739227 00000100:00000001:2.0:1788204611.817655:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.817656:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204611.817657:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817659:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204611.817661:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204611/real 1788204611] req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204611.817665:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204611.817666:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204611.817667:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204611.817669:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204611.817670:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204611.817671:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204611.817673:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204611.817678:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817682:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204611.817683:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817684:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.817684:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204611.817686:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817690:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204611.817692:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817696:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204611.817698:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817698:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.817699:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817700:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204611.817702:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204611.817703:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204611.817705:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204611.817706:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204611.817708:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204611.817712:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880064:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204611.817714:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204611.817716:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817720:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204611.817721:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817723:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204611.817724:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204611.817726:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204611.817729:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204611.817731:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204611.817733:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204611.817735:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817736:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204611.817739:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204611.817742:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204611.817745:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204611.817747:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204611.817751:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204611.817753:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880192, portal 4 00000100:00000001:2.0:1788204611.817755:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204611.817756:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204611.817762:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817768:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204611.817770:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204611.817772:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204611.817775:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205880192, offset 0 00000400:00000200:2.0:1788204611.817778:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204611.817782:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.817785:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204611.817787:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204611.817789:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204611.817790:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204611.817791:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204611.817793:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58732:58732:256:4294967295] 192.168.201.151@tcp LPNI seq info [58712:58712:8:4294967295] 00000400:00000200:2.0:1788204611.817795:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.817800:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204611.817802:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.817804:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204611.817807:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204611.817808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204611.817811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204611.817812:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1788204611.817815:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.817816:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204611.817818:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204611.817820:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204611.817822:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204611.817822:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204611.817824:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204611.817827:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817831:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.817833:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817837:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817838:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204611.817840:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204611.817842:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204611.817843:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204611.817845:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300739113 00000100:00000001:2.0:1788204611.817846:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.817847:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300739113 00000100:00000001:2.0:1788204611.817849:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.817850:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204611.817851:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.817852:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204611.817854:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204611/real 1788204611] req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204611.817858:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204611.817859:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204611.817859:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204611.817861:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204611.817862:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204611.817864:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204611.817867:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204611.817875:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817886:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204611.817888:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817889:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.817891:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204611.817894:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817901:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204611.817904:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817909:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204611.817910:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817911:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.817912:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.817913:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204611.817914:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204611.817916:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204611.817917:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204611.817918:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204611.817921:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204611.817925:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204611.817927:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204611.817929:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817933:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204611.817934:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817936:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204611.817937:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204611.817938:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204611.817940:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204611.817943:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204611.817944:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204611.817946:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204611.817947:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204611.817951:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204611.817952:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204611.817955:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204611.817957:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204611.817960:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204611.817963:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880320, portal 4 00000100:00000001:2.0:1788204611.817964:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204611.817965:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204611.817971:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.817977:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204611.817979:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204611.817981:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204611.817983:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205880320, offset 0 00000400:00000200:2.0:1788204611.817987:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204611.817990:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.817994:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204611.817996:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204611.817998:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204611.818000:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204611.818001:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204611.818003:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58733:58733:256:4294967295] 192.168.201.151@tcp LPNI seq info [58713:58713:8:4294967295] 00000400:00000200:2.0:1788204611.818005:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204611.818009:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204611.818012:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.818014:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204611.818016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204611.818018:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1788204611.818021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204611.818022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1788204611.818024:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204611.818025:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204611.818027:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204611.818029:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204611.818031:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204611.818032:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204611.818033:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204611.818036:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818040:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.818042:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.818047:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204611.818049:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204611.818050:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204611.818052:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204611.818053:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300739119 00000100:00000001:2.0:1788204611.818055:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.818056:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300739119 00000100:00000001:2.0:1788204611.818057:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300743169 00000100:00000001:2.0:1788204611.818058:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204611.818059:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.818060:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204611.818062:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204611/real 1788204611] req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204611.818067:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204611.818068:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204611.818069:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204611.818070:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204611.818071:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204611.818072:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204611.818074:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204611.818078:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818082:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204611.818083:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818084:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.818084:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204611.818086:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818089:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204611.818091:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818095:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204611.818096:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818097:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204611.818097:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818098:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204611.818100:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204611.818101:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204611.818102:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204611.818104:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204611.818106:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204611.818111:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204611.818113:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818114:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.818115:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205880064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818119:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204611.818120:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204611.818121:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204611.818122:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204611.818123:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204611.818124:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204611.818125:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204611.818126:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:2.0:1788204611.818128:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204611.818128:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204611.818129:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204611.818131:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204611.818131:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818132:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.818134:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205880192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818138:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204611.818138:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204611.818139:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204611.818140:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204611.818141:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204611.818141:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204611.818142:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204611.818143:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:2.0:1788204611.818144:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204611.818145:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204611.818146:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204611.818147:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204611.818148:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818149:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204611.818150:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205880320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204611.818153:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204611.818154:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204611.818155:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204611.818156:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204611.818156:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204611.818157:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204611.818158:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204611.818159:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:2.0:1788204611.818159:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204611.818160:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204611.818161:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204611.818162:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204611.818162:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818163:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.818166:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204611.818166:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.818167:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204611.818169:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008426:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204612.008432:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204612.008434:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204612.008437:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008441:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204612.008442:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008443:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008445:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008448:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204612.008449:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008452:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204612.008453:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204612.008454:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204612.008455:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008456:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204612.008456:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008457:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008459:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204612.008461:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204612.008462:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204612.008463:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008464:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204612.008464:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008465:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204612.008466:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204612.008514:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204612.008517:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204612.008519:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204612.328480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204612.328487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204612.328489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204612.328494:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204612.328497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204612.328500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204612.328502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204612.328504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204612.328506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204612.776468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204612.776473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204612.776475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204612.776479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204612.776488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204612.776491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204612.776493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204612.776495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204612.776497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204612.840464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204612.840472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204612.840475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204612.840480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204612.840485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204612.840491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204612.840500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204612.840503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204612.840505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204613.032505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204613.032512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204613.032518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204613.032519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204613.032523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204613.032524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204613.032527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204613.032528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204613.032546:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204613.032549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204613.032551:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204613.352470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204613.352476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204613.352479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204613.352483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204613.352486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204613.352490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204613.352493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204613.352496:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204613.352498:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204613.800441:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204613.800447:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204613.800448:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204613.800452:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204613.800455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204613.800457:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204613.800459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204613.800461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204613.800462:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204613.864449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204613.864453:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204613.864457:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204613.864463:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204613.864468:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204613.864472:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204613.864478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204613.864481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204613.864483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204614.056445:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204614.056451:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204614.056455:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204614.056456:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204614.056457:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204614.056458:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204614.056460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204614.056461:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204614.056511:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204614.056516:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204614.056519:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204614.376458:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204614.376463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204614.376465:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204614.376469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204614.376471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204614.376474:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.376476:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204614.376478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.376480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204614.824459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204614.824464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204614.824465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204614.824469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204614.824471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204614.824474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204614.824476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204614.824477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204614.824479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204614.888475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204614.888480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204614.888483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204614.888490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204614.888495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204614.888501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.888514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204614.888519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.888523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204615.080520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204615.080527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204615.080533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204615.080534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204615.080538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204615.080539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204615.080542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204615.080543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204615.080560:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204615.080562:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204615.080565:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204615.400443:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204615.400448:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204615.400450:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204615.400453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204615.400455:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204615.400458:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204615.400461:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204615.400463:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204615.400466:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204615.848440:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204615.848445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204615.848447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204615.848451:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204615.848454:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204615.848457:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204615.848459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204615.848462:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204615.848465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204615.912435:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204615.912441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204615.912444:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204615.912450:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204615.912456:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204615.912460:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204615.912469:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204615.912472:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204615.912474:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204616.104478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204616.104486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204616.104492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204616.104493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204616.104496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204616.104498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204616.104500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204616.104502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204616.104567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204616.104571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204616.104573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204616.424472:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300743169, at 4300743680 00000100:00000001:2.0:1788204616.424472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204616.424479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788204616.424481:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300743169, at 4300743680 00000100:00000001:2.0:1788204616.424484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204616.424487:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00100000:2.0:1788204616.424490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204616.424491:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000001:2.0:1788204616.424497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204616.424501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.424505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.424508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.424512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204616.872461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204616.872466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204616.872467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204616.872472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204616.872475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204616.872479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204616.872481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204616.872484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204616.872486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.936472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204616.936473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204616.936480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204616.936480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204616.936484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.936485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204616.936490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204616.936491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204616.936494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204616.936496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204616.936498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204616.936500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.936506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.936508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204616.936512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.936512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204616.936516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204616.936520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.936666:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204616.936669:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204616.936671:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204616.936676:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936684:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.936686:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.936689:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204616.936691:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204616.936693:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204616.936697:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205880448:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204616.936701:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204616.936705:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204616.936708:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.936710:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204616.936715:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204616.936717:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204616.936722:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204616.936725:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204616.936731:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204616.936737:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880448, portal 25 00000100:00000001:2.0:1788204616.936739:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204616.936741:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:2.0:1788204616.936746:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936753:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204616.936756:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204616.936758:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204616.936761:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205880448, offset 0 00000400:00000200:2.0:1788204616.936767:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204616.936775:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.936781:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204616.936786:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204616.936790:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204616.936794:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204616.936796:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204616.936800:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58734:58734:256:4294967295] 192.168.201.151@tcp LPNI seq info [58714:58714:8:4294967295] 00000400:00000200:2.0:1788204616.936804:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.936813:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204616.936819:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.936823:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204616.936829:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204616.936833:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788204616.936838:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204616.936842:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1788204616.936846:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.936848:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204616.936854:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204616.936858:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204616.936863:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204616.936865:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204616.936869:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204616.936873:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936882:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.936885:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936892:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.936894:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204616.936899:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204616.936903:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204616.936906:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300748289 00000100:00000010:2.0:1788204616.936910:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1b40. 00000100:00000001:2.0:1788204616.936916:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204616.936922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300748289 00000100:00000010:2.0:1788204616.936925:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1840. 00000100:00000001:2.0:1788204616.936928:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204616.936932:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204616.936934:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.936936:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204616.936940:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204616/real 1788204616] req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204616.936948:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204616.936950:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204616.936952:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204616.936954:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204616.936956:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204616.936958:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204616.936961:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204616.936970:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936977:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204616.936978:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.936980:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.936982:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204616.936986:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.936994:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204616.936998:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937006:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204616.937008:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937010:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937011:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937013:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204616.937015:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204616.937018:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204616.937021:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204616.937023:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204616.937028:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204616.937036:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205880448:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204616.937040:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937041:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937044:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205880448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937050:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204616.937053:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937054:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204616.937057:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204616.937059:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204616.937061:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204616.937063:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204616.937065:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:2.0:1788204616.937068:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204616.937069:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204616.937071:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:2.0:1788204616.937074:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204616.937075:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937077:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937084:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204616.937086:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937088:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204616.937089:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204616.937091:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204616.937095:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937102:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.937103:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937104:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204616.937106:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204616.937107:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204616.937109:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880576:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204616.937111:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204616.937112:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204616.937113:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937114:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937116:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204616.937117:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204616.937119:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204616.937121:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204616.937123:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204616.937125:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880576, portal 10 00000100:00000001:2.0:1788204616.937126:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204616.937127:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000040:2.0:1788204616.937131:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937135:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204616.937136:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204616.937138:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204616.937139:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205880576, offset 0 00000400:00000200:2.0:1788204616.937141:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937146:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204616.937168:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204616.937170:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204616.937172:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204616.937173:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937176:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58735:58735:256:4294967295] 192.168.201.151@tcp LPNI seq info [58715:58715:8:4294967295] 00000400:00000200:2.0:1788204616.937179:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937183:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204616.937186:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937188:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204616.937191:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204616.937193:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788204616.937195:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204616.937197:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1788204616.937199:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937200:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204616.937202:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204616.937205:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204616.937207:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204616.937208:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204616.937209:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204616.937211:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937216:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937218:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937222:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937223:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204616.937225:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204616.937227:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204616.937229:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204616.937231:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300744346 00000100:00000001:2.0:1788204616.937232:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937233:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300744346 00000100:00000001:2.0:1788204616.937235:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937236:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204616.937238:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937239:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204616.937241:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204616/real 1788204616] req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204616.937245:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204616.937246:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204616.937247:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204616.937248:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204616.937249:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204616.937250:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204616.937252:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204616.937257:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937260:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204616.937261:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937262:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937263:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204616.937265:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937268:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204616.937270:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937273:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204616.937275:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937276:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937277:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937278:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204616.937280:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204616.937283:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204616.937286:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204616.937288:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204616.937292:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204616.937300:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880576:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204616.937304:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204616.937307:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937313:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.937315:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937317:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204616.937318:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204616.937320:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204616.937323:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880704:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204616.937326:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204616.937327:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204616.937329:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937331:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937334:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204616.937335:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204616.937339:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204616.937342:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204616.937345:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204616.937348:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880704, portal 4 00000100:00000001:2.0:1788204616.937349:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204616.937351:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:2.0:1788204616.937356:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937379:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204616.937382:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204616.937384:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204616.937386:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205880704, offset 0 00000400:00000200:2.0:1788204616.937388:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937391:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937393:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204616.937395:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204616.937397:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204616.937398:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204616.937400:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937401:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58736:58736:256:4294967295] 192.168.201.151@tcp LPNI seq info [58716:58716:8:4294967295] 00000400:00000200:2.0:1788204616.937404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937407:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204616.937410:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937412:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204616.937414:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204616.937416:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788204616.937418:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204616.937420:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1788204616.937422:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937423:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204616.937425:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204616.937427:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204616.937429:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204616.937430:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204616.937432:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204616.937434:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937439:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937440:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937444:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937445:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204616.937447:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204616.937449:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204616.937451:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204616.937452:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300744233 00000100:00000001:2.0:1788204616.937453:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937455:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300744233 00000100:00000001:2.0:1788204616.937456:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937457:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204616.937458:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937459:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204616.937461:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204616/real 1788204616] req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204616.937466:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204616.937466:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204616.937467:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204616.937469:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204616.937470:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204616.937471:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204616.937473:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204616.937477:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937481:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204616.937482:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937483:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937483:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204616.937485:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937488:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204616.937490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937495:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204616.937497:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937497:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937498:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937499:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204616.937500:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204616.937502:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204616.937503:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204616.937504:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204616.937506:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204616.937510:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880704:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204616.937512:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204616.937514:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937517:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204616.937518:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937519:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204616.937520:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204616.937521:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204616.937523:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204616.937525:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204616.937526:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204616.937526:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204616.937527:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937529:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204616.937530:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204616.937532:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204616.937534:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204616.937535:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204616.937537:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880832, portal 4 00000100:00000001:2.0:1788204616.937538:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204616.937539:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:2.0:1788204616.937542:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937546:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204616.937547:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204616.937549:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204616.937550:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205880832, offset 0 00000400:00000200:2.0:1788204616.937552:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937553:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937555:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204616.937556:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204616.937558:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204616.937559:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204616.937561:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204616.937562:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58737:58737:256:4294967295] 192.168.201.151@tcp LPNI seq info [58717:58717:8:4294967295] 00000400:00000200:2.0:1788204616.937565:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204616.937568:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204616.937570:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937571:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204616.937573:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204616.937575:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1788204616.937577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204616.937579:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1788204616.937581:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204616.937583:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204616.937584:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204616.937586:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204616.937587:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204616.937588:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204616.937590:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204616.937592:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937596:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937598:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937601:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937602:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204616.937603:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204616.937604:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204616.937606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204616.937607:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300744239 00000100:00000001:2.0:1788204616.937608:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937610:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300744239 00000100:00000001:2.0:1788204616.937611:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300748289 00000100:00000001:2.0:1788204616.937612:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204616.937613:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937614:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204616.937616:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204616/real 1788204616] req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204616.937620:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204616.937620:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204616.937621:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204616.937622:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204616.937623:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204616.937624:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204616.937625:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204616.937630:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937637:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204616.937638:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937639:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937640:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204616.937641:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937645:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204616.937647:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937650:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204616.937651:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937652:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204616.937653:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937653:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204616.937655:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204616.937656:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204616.937657:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204616.937659:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204616.937660:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204616.937664:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205880832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204616.937666:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937667:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937669:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205880576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937673:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204616.937674:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937675:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204616.937677:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204616.937678:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204616.937679:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204616.937680:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204616.937681:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:2.0:1788204616.937682:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204616.937683:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204616.937684:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204616.937686:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204616.937687:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937688:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937690:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205880704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937694:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204616.937695:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937697:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204616.937699:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204616.937700:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204616.937701:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204616.937703:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204616.937704:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:2.0:1788204616.937707:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204616.937708:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204616.937709:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:2.0:1788204616.937711:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204616.937712:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937713:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204616.937717:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205880832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204616.937723:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204616.937724:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204616.937726:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204616.937728:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204616.937729:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204616.937730:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204616.937733:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204616.937734:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:2.0:1788204616.937736:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204616.937737:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204616.937738:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:2.0:1788204616.937740:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204616.937741:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937743:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.937745:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204616.937746:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.937748:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204616.937749:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204617.128509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204617.128516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204617.128521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204617.128523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204617.128526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204617.128527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204617.128530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204617.128532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204617.128547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204617.128549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204617.128552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204617.448427:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204617.448432:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204617.448434:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204617.448437:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204617.448440:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204617.448442:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204617.448444:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204617.448446:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204617.448449:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204617.896418:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204617.896422:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204617.896424:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204617.896427:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204617.896429:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204617.896431:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204617.896433:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204617.896434:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204617.896436:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.960436:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204617.960442:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204617.960444:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204617.960445:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1788204617.960448:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204617.960450:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204617.960452:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204617.960453:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.960457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204617.960458:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204617.960461:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204617.960464:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204617.960464:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204617.960468:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204617.960469:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204617.960473:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204617.960475:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204617.960477:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204618.152482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204618.152488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204618.152495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204618.152496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204618.152499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204618.152501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204618.152504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204618.152505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204618.152564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204618.152568:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204618.152571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204618.472485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204618.472492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204618.472494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204618.472499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204618.472502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204618.472506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.472509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204618.472510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.472513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204618.920476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204618.920482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204618.920484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204618.920489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204618.920492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204618.920496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204618.920499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204618.920501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204618.920504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.984463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204618.984463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204618.984470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204618.984470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204618.984472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.984474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204618.984478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204618.984480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204618.984483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204618.984485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204618.984486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204618.984489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.984490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204618.984493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.984495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.984496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204618.984499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204618.984503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204619.176515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204619.176520:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204619.176523:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204619.176525:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176526:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204619.176528:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176529:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176531:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204619.176533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204619.176535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204619.176536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204619.176549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204619.176550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204619.176552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204619.496480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204619.496485:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204619.496487:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204619.496490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204619.496492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204619.496495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204619.496497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204619.496499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204619.496501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204619.944463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204619.944470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204619.944471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204619.944475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204619.944477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204619.944481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204619.944482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204619.944484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204619.944486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204620.008467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204620.008468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204620.008475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204620.008476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204620.008478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204620.008480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204620.008485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204620.008486:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204620.008489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204620.008491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204620.008493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204620.008495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204620.008497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204620.008499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204620.008503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204620.008503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204620.008508:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204620.008511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204620.200458:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204620.200466:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204620.200472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204620.200473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204620.200476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204620.200477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204620.200481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204620.200483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204620.200540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204620.200543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204620.200547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204620.520470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204620.520475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204620.520477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204620.520480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204620.520483:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204620.520486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204620.520488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204620.520490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204620.520492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204620.968453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204620.968460:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204620.968463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204620.968467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204620.968471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204620.968474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204620.968476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204620.968478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204620.968480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204621.032466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204621.032466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204621.032473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204621.032473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204621.032476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204621.032477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204621.032483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:2.0:1788204621.032483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204621.032487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204621.032489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204621.032491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204621.032493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204621.032494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204621.032497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204621.032499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204621.032505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204621.032508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204621.032511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204621.224533:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204621.224536:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204621.224540:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224544:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204621.224547:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224549:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224551:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204621.224556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204621.224561:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204621.224562:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204621.224563:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224565:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204621.224566:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224568:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224569:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204621.224574:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204621.224575:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204621.224576:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224578:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204621.224579:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224581:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204621.224582:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204621.224600:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204621.224603:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204621.224606:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204621.544467:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300748289, at 4300748800 00000100:00000040:1.0:1788204621.544467:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300748289, at 4300748800 00000100:00000010:1.0:1788204621.544483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000010:0.0:1788204621.544485:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000001:1.0:1788204621.992868:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204621.992875:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204621.992877:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204621.992890:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204621.992893:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204621.992897:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204621.992899:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204621.992901:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204621.992904:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204622.056467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204622.056468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204622.056475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204622.056475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204622.056479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.056479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204622.056487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204622.056488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204622.056492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204622.056493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788204622.056493:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204622.056497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204622.056498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204622.056501:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204622.056502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204622.056504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204622.056507:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204622.056508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204622.056509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204622.056513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204622.056513:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204622.056518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.056520:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204622.056529:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6076 00000400:00000200:1.0:1788204622.056543:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204622.056549:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204622.056552:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204622.056556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056559:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204622.056562:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056564:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204622.056567:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204622.056569:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204622.056572:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204622.056574:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204622.056577:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204622.056579:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204622.056583:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056585:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204622.056587:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056589:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204622.056592:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204622.056595:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056598:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204622.056600:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204622.056603:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204622.056606:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.056608:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204622.056611:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204622.056626:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204622.056630:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204622.056634:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204622.056635:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.056638:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204622.056641:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204622.056644:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204622.056652:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204622.056659:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204622.056668:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204622.056678:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204622.056682:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204622.056687:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.056707:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204622.056710:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204622.056713:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204622.056714:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204622.056716:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204622.056719:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204622.056722:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6076 00000400:00000200:1.0:1788204622.056726:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204622.056730:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204622.056732:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788204622.056733:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204622.056737:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788204622.056738:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.056740:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788204622.056742:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204622.056748:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:2.0:1788204622.056750:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.056752:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204622.056757:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204622.056761:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204622.056765:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204622.056766:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204622.056769:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204622.056770:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:2.0:1788204622.056772:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1788204622.056775:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204622.056776:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788204622.056780:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204622.056781:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204622.056787:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205880960:192.168.201.151@tcp:250:kworker.0 00000100:00000040:1.0:1788204622.056790:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.056792:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788204622.056795:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788204622.056796:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1788204622.056800:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:2.0:1788204622.056800:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.056805:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788204622.056806:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204622.056811:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000010:2.0:1788204622.056811:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204622.056815:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204622.056818:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056824:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.056824:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788204622.056828:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1788204622.056830:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 02000000:00000001:1.0:1788204622.056834:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1788204622.056836:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 02000000:00000001:1.0:1788204622.056838:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.056842:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:2.0:1788204622.056843:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205880960, portal 25 00000100:00000001:2.0:1788204622.056846:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:1.0:1788204622.056847:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788204622.056847:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000008:00000001:1.0:1788204622.056853:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204622.056855:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.056856:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.056864:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1788204622.056865:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000100:00000040:2.0:1788204622.056868:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:1.0:1788204622.056871:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:2.0:1788204622.056872:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 02000000:00000001:1.0:1788204622.056877:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1788204622.056877:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205880960, offset 0 02000000:00000001:1.0:1788204622.056881:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.056884:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204622.056885:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204622.056888:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204622.056890:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.056893:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:1.0:1788204622.056895:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000400:00000200:2.0:1788204622.056899:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:1.0:1788204622.056900:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.056905:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:1.0:1788204622.056910:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204622.056910:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204622.056915:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204622.056920:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204622.056926:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58738:58738:256:4294967295] 192.168.201.151@tcp LPNI seq info [58718:58718:8:4294967295] 00000100:00000001:1.0:1788204622.056928:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204622.056931:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204622.056934:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204622.056943:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204622.056943:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204622.056951:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.056959:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000100:00000001:1.0:1788204622.056961:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204622.056966:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1788204622.056970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:1.0:1788204622.056972:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204622.056976:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.0:1788204622.056976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00080000:1.0:1788204622.056979:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204622.056984:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.1:1788204622.056986:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:1.0:1788204622.056990:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6076 00000800:00000200:2.1:1788204622.056992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000400:00000200:1.0:1788204622.056998:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000100:2.0:1788204622.057001:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788204622.057005:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000010:2.0:1788204622.057005:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000100:00080000:1.0:1788204622.057010:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204622.057013:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1788204622.057014:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.057019:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204622.057020:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1788204622.057025:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057026:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204622.057030:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:1.0:1788204622.057031:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204622.057034:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1788204622.057035:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204622.057038:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000200:2.0:1788204622.057040:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204622.057041:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204622.057046:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204622.057051:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204622.057056:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:2.0:1788204622.057067:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057073:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204622.057076:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.057082:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204622.057088:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.057088:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057092:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204622.057093:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204622.057098:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000001:2.0:1788204622.057099:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204622.057102:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057107:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204622.057108:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057111:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300753409 02000000:00000001:1.0:1788204622.057112:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:2.0:1788204622.057116:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ba0. 02000000:00000001:1.0:1788204622.057118:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204622.057122:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057122:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788204622.057124:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204622.057128:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:2.0:1788204622.057129:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300753409 00000008:00000001:1.0:1788204622.057133:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204622.057134:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1300. 00000100:00000001:2.0:1788204622.057138:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788204622.057139:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204622.057145:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788204622.057148:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:2.0:1788204622.057148:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204622.057153:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204622.057154:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204622.057156:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788204622.057157:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.057162:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1788204622.057162:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204622.057166:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204622.057169:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204622.057173:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:1.0:1788204622.057176:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000400:00000001:2.0:1788204622.057177:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204622.057180:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000100:00000001:1.0:1788204622.057182:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.057184:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204622.057188:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00100000:1.0:1788204622.057191:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204622.057193:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204622.057199:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.057206:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204622.057211:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204622.057216:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204622.057220:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057232:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204622.057235:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204622.057237:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057239:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1788204622.057242:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204622.057243:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204622.057249:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1788204622.057251:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.057253:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204622.057256:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204622.057263:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204622.057266:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204622.057269:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6076 00000100:00000040:2.0:1788204622.057275:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204622.057278:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204622.057283:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204622.057289:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204622.057295:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.057297:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204622.057299:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204622.057302:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204622.057304:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.057305:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.057307:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204622.057310:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204622.057310:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204622.057313:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1788204622.057316:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:2.0:1788204622.057316:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1788204622.057320:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00080000:2.0:1788204622.057322:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000020:00000001:1.0:1788204622.057324:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204622.057326:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:1.0:1788204622.057328:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204622.057334:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204622.057334:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204622.057339:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204622.057348:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204622.057352:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788204622.057353:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205880960:192.168.201.151@tcp:250:kworker.0 00000100:00000040:1.0:1788204622.057357:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204622.057361:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057384:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057388:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205880960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057394:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204622.057397:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204622.057399:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 00000100:00000001:1.0:1788204622.057402:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204622.057403:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204622.057406:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788204622.057408:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000020:00000040:2.0:1788204622.057409:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204622.057414:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:1.0:1788204622.057415:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000010:2.0:1788204622.057417:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204622.057420:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.057423:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204622.057424:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204622.057426:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1788204622.057429:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1788204622.057429:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:2.0:1788204622.057431:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1788204622.057435:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057435:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057437:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204622.057440:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204622.057444:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204622.057447:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.057448:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057452:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788204622.057453:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204622.057455:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204622.057459:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204622.057460:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204622.057463:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057463:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204622.057467:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204622.057468:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788204622.057470:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204622.057471:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788204622.057475:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204622.057479:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788204622.057482:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788204622.057488:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057489:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204622.057492:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.057494:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204622.057496:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788204622.057498:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204622.057499:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204622.057504:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881088:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204622.057510:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204622.057512:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204622.057514:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.057517:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788204622.057519:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1788204622.057521:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204622.057524:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204622.057525:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.057529:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204622.057531:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204622.057534:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204622.057536:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881088, portal 10 00000100:00000001:2.0:1788204622.057538:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1788204622.057539:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057540:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:2.0:1788204622.057547:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204622.057561:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057561:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204622.057564:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204622.057567:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204622.057568:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205881088, offset 0 00000100:00000040:1.0:1788204622.057570:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6087) 00000400:00000200:2.0:1788204622.057573:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057577:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.057580:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204622.057584:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204622.057587:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204622.057590:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204622.057592:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057595:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58739:58739:256:4294967295] 192.168.201.151@tcp LPNI seq info [58719:58719:8:4294967295] 00000400:00000200:2.0:1788204622.057599:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.057605:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204622.057609:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.057613:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204622.057616:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204622.057619:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204622.057624:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204622.057638:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1788204622.057642:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.057644:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204622.057647:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204622.057650:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204622.057654:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204622.057655:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204622.057667:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204622.057671:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057679:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057681:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057689:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057690:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204622.057693:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204622.057696:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204622.057698:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204622.057700:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300749467 00000100:00000001:2.0:1788204622.057702:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.057703:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300749467 00000100:00000001:2.0:1788204622.057704:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.057705:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204622.057707:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057708:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204622.057710:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204622.057714:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204622.057715:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204622.057716:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204622.057717:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204622.057718:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204622.057719:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204622.057721:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204622.057726:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057729:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204622.057730:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057731:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.057731:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204622.057733:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057737:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204622.057740:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057743:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204622.057745:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057745:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.057746:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057747:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204622.057748:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204622.057750:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204622.057751:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204622.057752:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204622.057754:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204622.057759:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881088:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204622.057761:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204622.057763:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057767:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204622.057768:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.057769:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204622.057770:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204622.057771:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204622.057772:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881216:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204622.057774:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204622.057775:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204622.057776:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.057777:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204622.057780:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204622.057781:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.057783:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204622.057784:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204622.057786:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204622.057788:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881216, portal 4 00000100:00000001:2.0:1788204622.057789:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204622.057790:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:2.0:1788204622.057793:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057797:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204622.057798:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204622.057799:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204622.057801:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205881216, offset 0 00000400:00000200:2.0:1788204622.057803:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057805:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.057807:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204622.057809:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204622.057810:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204622.057812:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204622.057813:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204622.057815:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58740:58740:256:4294967295] 192.168.201.151@tcp LPNI seq info [58720:58720:8:4294967295] 00000400:00000200:2.0:1788204622.057817:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.057821:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204622.057824:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.057826:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204622.057828:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204622.057830:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204622.057832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204622.057834:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1788204622.057836:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.057838:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204622.057839:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204622.057841:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204622.057843:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204622.057844:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204622.057846:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204622.057847:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057852:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057854:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057857:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057858:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204622.057859:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204622.057861:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204622.057863:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204622.057864:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300749353 00000100:00000001:2.0:1788204622.057865:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.057867:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300749353 00000100:00000001:2.0:1788204622.057869:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.057871:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204622.057873:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.057875:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204622.057878:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204622.057885:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204622.057886:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204622.057887:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204622.057889:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204622.057890:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204622.057893:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204622.057895:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204622.057903:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057910:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204622.057911:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057913:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.057914:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204622.057917:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057924:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204622.057927:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057933:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204622.057935:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057936:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.057938:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057939:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204622.057941:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204622.057944:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204622.057947:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204622.057949:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204622.057952:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204622.057960:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881216:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204622.057963:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057965:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057968:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205881088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057972:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204622.057973:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204622.057975:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204622.057976:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204622.057977:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204622.057978:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204622.057979:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204622.057981:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:2.0:1788204622.057982:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204622.057983:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204622.057985:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:2.0:1788204622.057987:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204622.057988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.057990:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.057993:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205881216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.057998:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204622.057999:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204622.058001:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204622.058002:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204622.058003:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204622.058004:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204622.058006:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204622.058007:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:2.0:1788204622.058009:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204622.058010:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204622.058011:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:2.0:1788204622.058013:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204622.058014:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058016:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058019:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204622.058020:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.058022:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204622.058023:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204622.058025:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204622.058028:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058034:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204622.058035:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.058037:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204622.058039:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204622.058041:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204622.058043:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881344:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204622.058046:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204622.058048:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204622.058049:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204622.058051:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204622.058053:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204622.058055:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204622.058067:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204622.058070:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204622.058073:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204622.058076:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881344, portal 4 00000100:00000001:2.0:1788204622.058078:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204622.058079:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:2.0:1788204622.058084:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058088:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204622.058090:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204622.058092:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204622.058094:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205881344, offset 0 00000400:00000200:2.0:1788204622.058097:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204622.058101:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.058103:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204622.058106:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204622.058108:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204622.058110:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204622.058112:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204622.058115:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58741:58741:256:4294967295] 192.168.201.151@tcp LPNI seq info [58721:58721:8:4294967295] 00000400:00000200:2.0:1788204622.058119:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204622.058124:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204622.058128:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.058131:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204622.058135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204622.058137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1788204622.058141:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204622.058144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1788204622.058147:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204622.058149:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204622.058152:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204622.058155:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204622.058158:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204622.058159:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204622.058162:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204622.058165:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058170:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.058172:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058175:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.058176:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204622.058178:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204622.058180:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204622.058182:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204622.058184:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300749359 00000100:00000001:2.0:1788204622.058186:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.058188:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300749359 00000100:00000001:2.0:1788204622.058190:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300753409 00000100:00000001:2.0:1788204622.058192:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204622.058194:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.058196:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204622.058199:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204622.058206:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204622.058208:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204622.058209:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204622.058211:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204622.058213:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204622.058215:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204622.058218:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204622.058226:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058231:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204622.058232:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058234:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.058235:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204622.058238:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058244:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204622.058248:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058253:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204622.058255:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058256:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204622.058257:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058259:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204622.058261:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204622.058264:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204622.058267:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204622.058269:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204622.058272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204622.058279:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881344:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204622.058282:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058283:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204622.058286:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205881344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204622.058291:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204622.058293:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204622.058294:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204622.058296:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204622.058298:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204622.058299:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204622.058301:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204622.058302:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc400. 02000000:00000001:2.0:1788204622.058305:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204622.058306:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204622.058307:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:2.0:1788204622.058309:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204622.058309:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058311:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204622.058314:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204622.058315:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.058317:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204622.058318:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204622.248449:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204622.248455:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204622.248460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204622.248461:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204622.248463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204622.248464:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204622.248466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204622.248467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204622.248522:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204622.248526:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204622.248528:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204623.016439:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204623.016444:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204623.016445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204623.016449:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204623.016455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204623.016458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204623.016460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204623.016461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204623.016463:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204623.080450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204623.080450:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204623.080456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204623.080458:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204623.080463:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204623.080465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204623.080466:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204623.080470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204623.080472:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204623.080473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204623.080477:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204623.080478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204623.080479:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204623.080482:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204623.080485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.080486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204623.080489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204623.080492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204623.272509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204623.272515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204623.272518:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204623.272519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204623.272521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204623.272522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204623.272524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204623.272524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204623.272537:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204623.272539:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204623.272541:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204624.040487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204624.040494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204624.040496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204624.040502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204624.040505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204624.040509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204624.040511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204624.040513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204624.040517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204624.104537:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204624.104544:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204624.104546:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204624.104551:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204624.104555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204624.104559:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204624.104561:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204624.104565:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204624.104568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204624.104691:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204624.104695:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204624.104696:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204624.104699:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204624.104701:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204624.104703:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204624.104705:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204624.104707:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204624.104709:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204624.296464:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204624.296471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204624.296477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204624.296478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204624.296482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204624.296483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204624.296487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204624.296487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204624.296547:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204624.296550:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204624.296553:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204625.064455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204625.064462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204625.064464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204625.064469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204625.064472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204625.064476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204625.064478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204625.064480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204625.064483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204625.128474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204625.128474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204625.128481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204625.128482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204625.128484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204625.128486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204625.128489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204625.128491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204625.128493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204625.128497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204625.128497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204625.128501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204625.128501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204625.128505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204625.128508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204625.128509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204625.128512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204625.128515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204625.320515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204625.320521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204625.320526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204625.320527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204625.320530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204625.320532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204625.320535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204625.320536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204625.320553:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204625.320555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204625.320557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204626.088488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204626.088495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204626.088497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204626.088502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204626.088505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204626.088508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204626.088510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204626.088519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204626.088522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204626.152452:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204626.152452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204626.152459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204626.152460:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204626.152464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204626.152464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204626.152471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204626.152472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204626.152478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204626.152478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204626.152484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204626.152484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204626.152490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204626.152493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204626.152496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204626.152496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204626.152500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204626.152502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204626.344469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204626.344474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204626.344477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204626.344478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204626.344480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204626.344482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204626.344484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204626.344485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204626.344533:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204626.344536:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204626.344540:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204626.664445:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300753409, at 4300753920 00000100:00000010:1.0:1788204626.664457:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000040:0.0:1788204626.664527:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300753409, at 4300753920 00000100:00000010:0.0:1788204626.664533:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000001:1.0:1788204627.112432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204627.112438:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204627.112439:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204627.112443:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204627.112445:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204627.112447:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204627.112449:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204627.112451:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204627.112453:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204627.176467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204627.176473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204627.176474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204627.176475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204627.176478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204627.176480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204627.176483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204627.176483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204627.176489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204627.176489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204627.176493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204627.176493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204627.176495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204627.176498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204627.176499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204627.176503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204627.176505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204627.176515:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204627.176519:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204627.176521:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204627.176523:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204627.176526:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204627.176532:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6082 00000400:00000200:0.0:1788204627.176545:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204627.176549:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204627.176551:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204627.176554:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176556:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.176557:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204627.176561:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204627.176563:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204627.176565:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204627.176567:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204627.176570:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204627.176572:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204627.176575:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176577:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.176579:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176581:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204627.176584:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204627.176587:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176590:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204627.176593:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204627.176596:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204627.176599:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176600:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204627.176603:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204627.176612:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204627.176615:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204627.176619:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204627.176621:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176623:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204627.176626:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204627.176629:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204627.176638:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204627.176645:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204627.176654:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204627.176664:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204627.176668:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204627.176672:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204627.176692:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204627.176695:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204627.176698:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204627.176700:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204627.176701:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204627.176704:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204627.176707:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6082 00000400:00000200:0.0:1788204627.176711:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204627.176714:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204627.176716:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204627.176719:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176721:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.176723:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176726:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204627.176728:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204627.176729:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204627.176731:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204627.176732:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1788204627.176733:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204627.176739:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204627.176739:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204627.176742:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204627.176743:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204627.176749:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176753:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204627.176754:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204627.176758:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176763:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204627.176769:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204627.176770:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204627.176774:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204627.176775:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176780:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.176780:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204627.176785:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204627.176785:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204627.176790:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204627.176792:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788204627.176796:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205881472:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204627.176797:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176802:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204627.176805:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788204627.176807:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204627.176811:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1788204627.176812:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.176815:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176816:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.176821:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1788204627.176823:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204627.176827:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000010:2.0:1788204627.176828:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:0.0:1788204627.176832:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204627.176833:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204627.176836:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.176839:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1788204627.176842:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204627.176843:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204627.176848:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204627.176849:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 02000000:00000010:0.0:1788204627.176854:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000400:00000010:2.0:1788204627.176857:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204627.176859:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204627.176865:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881472, portal 25 00000100:00100000:0.0:1788204627.176869:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204627.176870:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204627.176873:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204627.176882:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204627.176883:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204627.176887:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204627.176896:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.176898:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204627.176904:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204627.176908:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000001:0.0:1788204627.176915:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204627.176915:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205881472, offset 0 00000100:00080000:0.0:1788204627.176920:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204627.176926:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1788204627.176927:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176930:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204627.176933:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204627.176937:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204627.176939:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204627.176943:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6082 00000400:00000200:0.0:1788204627.176947:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204627.176951:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204627.176954:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204627.176958:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788204627.176959:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.176964:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.176967:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204627.176968:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204627.176983:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204627.176988:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1788204627.176988:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:0.0:1788204627.176992:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204627.176994:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204627.176996:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788204627.176997:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204627.177002:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204627.177002:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788204627.177008:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204627.177009:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58742:58742:256:4294967295] 192.168.201.151@tcp LPNI seq info [58722:58722:8:4294967295] 00000100:00000040:0.0:1788204627.177013:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177018:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204627.177018:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204627.177023:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177027:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204627.177032:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204627.177036:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:0.0:1788204627.177040:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177044:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204627.177047:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1788204627.177049:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204627.177051:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204627.177055:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.177057:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:2.0:1788204627.177057:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000008:00100000:0.0:1788204627.177062:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204627.177067:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204627.177068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204627.177071:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204627.177074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000100:00000010:0.0:1788204627.177076:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204627.177081:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.1:1788204627.177086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204627.177087:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204627.177089:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.177092:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204627.177093:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204627.177095:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204627.177098:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 02000000:00000010:0.0:1788204627.177099:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204627.177106:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788204627.177108:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177112:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000100:00100000:0.0:1788204627.177114:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204627.177120:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204627.177125:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204627.177130:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204627.177131:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1788204627.177138:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204627.177139:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204627.177144:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204627.177150:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204627.177152:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204627.177157:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1788204627.177158:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.177160:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204627.177163:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204627.177165:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204627.177170:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204627.177174:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6082 00000400:00000200:0.0:1788204627.177179:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204627.177179:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1788204627.177186:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:2.0:1788204627.177188:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.177189:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204627.177193:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177197:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.177201:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177204:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204627.177208:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204627.177210:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204627.177211:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204627.177214:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204627.177215:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1788204627.177216:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204627.177220:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204627.177221:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204627.177223:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204627.177223:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.177228:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204627.177231:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177232:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204627.177233:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204627.177235:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204627.177236:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204627.177237:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204627.177239:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204627.177240:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204627.177242:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177244:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300758529 00000100:00000001:0.0:1788204627.177245:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204627.177248:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204627.177251:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.177252:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204627.177252:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ba0. 00000100:00000010:0.0:1788204627.177258:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204627.177261:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204627.177261:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788204627.177266:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204627.177266:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204627.177268:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204627.177269:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204627.177269:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300758529 02000000:00000001:0.0:1788204627.177272:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204627.177275:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1300. 02000000:00000010:0.0:1788204627.177277:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:2.0:1788204627.177278:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204627.177282:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177283:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204627.177287:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204627.177290:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204627.177290:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204627.177296:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204627.177305:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788204627.177308:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204627.177310:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204627.177312:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1788204627.177316:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204627.177317:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204627.177321:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204627.177324:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204627.177328:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788204627.177329:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204627.177334:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6092) 00000100:00000200:2.0:1788204627.177334:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204627.177343:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177348:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204627.177349:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177350:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177351:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204627.177354:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177358:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204627.177362:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177386:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204627.177388:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177389:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177390:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177391:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204627.177393:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204627.177395:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204627.177397:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204627.177399:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204627.177402:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204627.177407:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205881472:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204627.177410:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177411:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177413:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205881472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177416:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204627.177418:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177420:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204627.177422:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204627.177423:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204627.177424:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204627.177425:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204627.177427:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:2.0:1788204627.177429:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204627.177431:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204627.177432:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204627.177434:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204627.177435:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177436:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177441:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204627.177442:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177451:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204627.177452:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204627.177453:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204627.177455:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177459:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204627.177460:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177461:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204627.177463:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204627.177464:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204627.177466:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881600:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204627.177468:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204627.177469:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204627.177470:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177471:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177473:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204627.177474:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204627.177477:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204627.177478:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204627.177483:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204627.177485:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881600, portal 10 00000100:00000001:2.0:1788204627.177486:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204627.177487:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204627.177491:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177495:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204627.177497:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204627.177498:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204627.177499:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205881600, offset 0 00000400:00000200:2.0:1788204627.177502:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177504:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177506:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204627.177508:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204627.177510:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204627.177511:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204627.177513:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177515:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58743:58743:256:4294967295] 192.168.201.151@tcp LPNI seq info [58723:58723:8:4294967295] 00000400:00000200:2.0:1788204627.177518:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177522:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204627.177524:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177526:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204627.177528:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204627.177530:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204627.177533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204627.177534:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204627.177536:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177538:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204627.177540:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204627.177542:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204627.177543:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204627.177545:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204627.177546:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204627.177548:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177553:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177555:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177558:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177559:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204627.177561:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204627.177563:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204627.177565:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204627.177566:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300754587 00000100:00000001:2.0:1788204627.177567:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177568:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300754587 00000100:00000001:2.0:1788204627.177571:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177572:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204627.177574:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177575:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204627.177577:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204627.177581:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204627.177581:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204627.177582:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204627.177583:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204627.177584:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204627.177585:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204627.177587:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204627.177591:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177595:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204627.177596:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177597:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177598:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204627.177600:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177603:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204627.177605:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177615:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204627.177616:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177617:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177618:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177619:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204627.177621:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204627.177622:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204627.177624:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204627.177625:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204627.177628:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204627.177632:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881600:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204627.177634:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204627.177636:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177639:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204627.177640:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177641:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204627.177642:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204627.177644:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204627.177645:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881728:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204627.177647:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204627.177648:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204627.177649:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177650:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177652:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204627.177653:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204627.177654:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204627.177656:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204627.177658:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204627.177659:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881728, portal 4 00000100:00000001:2.0:1788204627.177661:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204627.177661:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204627.177665:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177669:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204627.177670:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204627.177671:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204627.177673:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205881728, offset 0 00000400:00000200:2.0:1788204627.177675:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177676:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177678:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204627.177680:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204627.177681:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204627.177682:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204627.177684:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177686:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58744:58744:256:4294967295] 192.168.201.151@tcp LPNI seq info [58724:58724:8:4294967295] 00000400:00000200:2.0:1788204627.177688:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177692:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204627.177694:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177696:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204627.177698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204627.177699:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204627.177702:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204627.177703:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204627.177705:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177707:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204627.177708:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204627.177710:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204627.177712:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204627.177713:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204627.177714:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204627.177716:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177721:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177722:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177730:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177731:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204627.177732:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204627.177734:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204627.177735:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204627.177737:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300754473 00000100:00000001:2.0:1788204627.177738:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177740:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300754473 00000100:00000001:2.0:1788204627.177741:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177742:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204627.177743:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177744:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204627.177746:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204627.177749:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204627.177750:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204627.177751:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204627.177752:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204627.177753:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204627.177754:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204627.177756:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204627.177760:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177764:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204627.177764:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177765:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177766:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204627.177768:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177772:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204627.177774:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177778:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204627.177779:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177780:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177781:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177782:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204627.177783:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204627.177784:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204627.177786:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204627.177787:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204627.177789:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204627.177792:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881728:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204627.177794:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204627.177796:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177799:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204627.177800:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177801:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204627.177802:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204627.177803:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204627.177805:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204627.177807:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204627.177808:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204627.177809:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204627.177810:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177812:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204627.177813:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204627.177814:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204627.177816:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204627.177817:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204627.177819:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881856, portal 4 00000100:00000001:2.0:1788204627.177820:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204627.177821:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204627.177824:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177827:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204627.177828:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204627.177829:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204627.177830:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205881856, offset 0 00000400:00000200:2.0:1788204627.177832:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177834:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177835:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204627.177837:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204627.177838:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204627.177840:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204627.177841:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204627.177843:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58745:58745:256:4294967295] 192.168.201.151@tcp LPNI seq info [58725:58725:8:4294967295] 00000400:00000200:2.0:1788204627.177845:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204627.177849:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204627.177851:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177852:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204627.177854:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204627.177855:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204627.177857:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204627.177859:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204627.177861:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204627.177863:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204627.177864:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204627.177866:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204627.177868:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204627.177868:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204627.177870:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204627.177871:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177875:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177877:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177880:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177881:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204627.177882:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204627.177884:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204627.177886:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204627.177887:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300754479 00000100:00000001:2.0:1788204627.177888:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177889:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300754479 00000100:00000001:2.0:1788204627.177890:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300758529 00000100:00000001:2.0:1788204627.177891:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204627.177892:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.177893:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204627.177895:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204627.177899:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204627.177899:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204627.177900:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204627.177901:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204627.177902:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204627.177904:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204627.177905:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204627.177910:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177913:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204627.177914:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177915:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177915:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204627.177917:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177920:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204627.177922:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177926:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204627.177927:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177928:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204627.177928:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177929:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204627.177930:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204627.177931:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204627.177933:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204627.177934:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204627.177936:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204627.177940:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205881856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204627.177941:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177943:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177945:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205881600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177949:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204627.177950:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177951:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204627.177952:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204627.177952:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204627.177953:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204627.177954:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204627.177955:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788204627.177957:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204627.177958:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204627.177958:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204627.177960:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204627.177961:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177962:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177964:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205881728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.177967:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204627.177968:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204627.177969:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204627.177983:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204627.177984:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204627.177985:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204627.177986:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204627.177987:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:2.0:1788204627.177989:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204627.177989:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204627.177990:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204627.177991:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204627.177992:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.177993:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204627.177996:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205881856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204627.178000:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204627.178001:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204627.178002:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204627.178003:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204627.178004:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204627.178004:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204627.178005:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204627.178006:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:2.0:1788204627.178007:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204627.178008:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204627.178008:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204627.178009:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204627.178010:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.178011:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204627.178013:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204627.178014:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.178015:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204627.178016:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204627.368486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204627.368491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204627.368495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204627.368496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204627.368498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204627.368498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204627.368500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204627.368501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204627.368512:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204627.368514:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204627.368516:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204628.136466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204628.136472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204628.136473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204628.136477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204628.136479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204628.136482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204628.136484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204628.136485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204628.136486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204628.200427:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204628.200434:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204628.200435:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204628.200440:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204628.200443:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204628.200447:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204628.200449:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204628.200451:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204628.200453:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204628.200473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204628.200477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204628.200479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204628.200483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204628.200486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204628.200488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204628.200491:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204628.200492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204628.200494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204628.392473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204628.392477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204628.392481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204628.392482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204628.392484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204628.392486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204628.392488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204628.392489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204628.392545:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204628.392549:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204628.392552:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204629.160460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204629.160465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204629.160466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204629.160469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204629.160471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204629.160474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204629.160476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204629.160477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204629.160479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204629.224461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204629.224467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204629.224468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204629.224472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204629.224472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204629.224477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204629.224478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204629.224481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204629.224483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204629.224484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204629.224486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204629.224488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204629.224489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204629.224491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204629.224492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204629.224493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204629.224497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204629.224499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204629.416475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204629.416481:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204629.416483:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204629.416485:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416487:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204629.416488:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416490:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416491:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204629.416494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204629.416495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204629.416496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204629.416507:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204629.416509:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204629.416511:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204630.184468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204630.184473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204630.184475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204630.184479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204630.184482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204630.184485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204630.184487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204630.184488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204630.184490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204630.248464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204630.248464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204630.248470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204630.248472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204630.248473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204630.248475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204630.248477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204630.248478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204630.248481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204630.248482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204630.248485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204630.248486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204630.248489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204630.248492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204630.248494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204630.248495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204630.248497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204630.248499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204630.440459:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204630.440465:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204630.440469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204630.440470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204630.440473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204630.440473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204630.440476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204630.440477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204630.440536:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204630.440539:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204630.440543:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204631.208493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204631.208499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204631.208501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204631.208507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204631.208510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204631.208514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204631.208516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204631.208519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204631.208522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204631.272460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204631.272460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204631.272467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204631.272467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204631.272470:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204631.272470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204631.272475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204631.272476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204631.272479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204631.272481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204631.272485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204631.272488:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204631.272490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204631.272491:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204631.272496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204631.272497:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204631.272498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204631.272501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204631.464494:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204631.464497:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204631.464500:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464504:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204631.464506:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464508:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464510:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204631.464521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204631.464526:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204631.464527:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204631.464529:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464530:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204631.464531:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464533:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464535:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204631.464539:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204631.464540:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204631.464541:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464543:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204631.464544:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464545:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204631.464547:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204631.464563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204631.464565:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204631.464568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204631.784466:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300758529, at 4300759040 00000100:00000040:1.0:1788204631.784466:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300758529, at 4300759040 00000100:00000010:1.0:1788204631.784480:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000010:0.0:1788204631.784481:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000001:1.0:1788204632.232485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204632.232493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204632.232495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204632.232500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204632.232503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204632.232507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204632.232509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204632.232512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204632.232515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204632.296472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204632.296479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204632.296479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204632.296483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204632.296489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204632.296491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204632.296492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204632.296497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204632.296497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204632.296500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204632.296503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204632.296505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204632.296508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204632.296510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204632.296514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204632.296515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204632.296518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204632.296552:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204632.296558:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204632.296559:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204632.296562:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204632.296565:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204632.296569:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6087 00000400:00000200:0.0:1788204632.296576:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204632.296580:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204632.296582:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204632.296585:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.296589:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296591:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204632.296594:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204632.296595:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204632.296597:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204632.296600:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204632.296601:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204632.296603:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204632.296606:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296607:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.296608:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296613:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204632.296615:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204632.296618:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296619:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204632.296621:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204632.296623:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204632.296625:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296627:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204632.296629:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204632.296636:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204632.296639:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204632.296641:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204632.296643:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296645:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204632.296647:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204632.296651:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204632.296660:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204632.296666:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204632.296675:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204632.296685:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204632.296689:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204632.296693:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204632.296711:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204632.296713:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204632.296715:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204632.296716:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204632.296717:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204632.296719:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204632.296721:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6087 00000400:00000200:0.0:1788204632.296724:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204632.296725:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204632.296727:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204632.296728:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296730:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.296731:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296733:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204632.296736:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204632.296738:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204632.296740:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204632.296742:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204632.296744:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204632.296746:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204632.296749:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296752:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204632.296752:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788204632.296757:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204632.296758:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204632.296760:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204632.296762:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1788204632.296764:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204632.296771:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:2.0:1788204632.296772:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204632.296774:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204632.296778:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204632.296783:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204632.296785:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788204632.296786:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296789:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204632.296790:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204632.296793:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204632.296796:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.296796:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204632.296800:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.296801:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1788204632.296803:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204632.296807:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204632.296807:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204632.296811:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204632.296813:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296815:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204632.296817:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1788204632.296817:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205881984:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204632.296819:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204632.296824:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:2.0:1788204632.296825:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204632.296829:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.296831:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1788204632.296835:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204632.296836:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.296842:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204632.296845:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204632.296848:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204632.296850:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 00000100:00000040:0.0:1788204632.296851:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204632.296856:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296862:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204632.296865:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204632.296866:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204632.296872:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:2.0:1788204632.296873:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00000001:0.0:1788204632.296874:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204632.296877:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204632.296892:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:2.0:1788204632.296892:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00080000:0.0:1788204632.296897:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6087 00000100:00000200:2.0:1788204632.296900:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205881984, portal 25 00000400:00000200:0.0:1788204632.296904:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204632.296904:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204632.296906:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000400:00000200:0.0:1788204632.296910:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204632.296914:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204632.296917:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204632.296920:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296925:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.296928:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296932:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204632.296936:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204632.296939:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000001:2.0:1788204632.296939:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788204632.296942:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204632.296944:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204632.296947:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204632.296947:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788204632.296950:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204632.296953:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296954:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.296955:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296956:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204632.296957:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204632.296959:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296961:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204632.296961:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 02000000:00000001:0.0:1788204632.296964:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204632.296967:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204632.296969:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.296970:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:2.0:1788204632.296970:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205881984, offset 0 00000008:00100000:0.0:1788204632.296974:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204632.296977:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.296979:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204632.296980:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204632.296985:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204632.296988:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204632.296990:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204632.296992:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.296992:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204632.296996:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204632.296997:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204632.296999:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297002:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000010:0.0:1788204632.297003:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204632.297007:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297012:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1788204632.297013:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204632.297019:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204632.297021:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204632.297023:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204632.297026:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:0.0:1788204632.297033:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297034:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1788204632.297035:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204632.297040:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1788204632.297042:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.297043:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204632.297045:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204632.297048:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204632.297051:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58746:58746:256:4294967295] 192.168.201.151@tcp LPNI seq info [58726:58726:8:4294967295] 00000100:00080000:0.0:1788204632.297052:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6087 00000400:00000200:0.0:1788204632.297059:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204632.297063:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204632.297064:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204632.297065:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204632.297069:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.297071:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204632.297072:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.297074:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204632.297078:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204632.297080:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000400:00000200:2.0:1788204632.297081:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:0.0:1788204632.297082:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204632.297085:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204632.297087:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204632.297089:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:2.0:1788204632.297091:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204632.297093:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.297096:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1788204632.297097:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000100:00000040:0.0:1788204632.297098:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204632.297102:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204632.297104:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204632.297108:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:2.0:1788204632.297108:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204632.297110:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204632.297113:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1788204632.297114:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 02000000:00000001:0.0:1788204632.297115:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204632.297120:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.297122:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.1:1788204632.297123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00100000:0.0:1788204632.297124:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000800:00000200:2.1:1788204632.297128:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000008:00000001:0.0:1788204632.297129:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204632.297132:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788204632.297136:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000010:0.0:1788204632.297137:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000800:00000010:2.0:1788204632.297138:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000020:00000040:0.0:1788204632.297140:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204632.297143:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204632.297144:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297144:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204632.297149:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204632.297151:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1788204632.297152:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1788204632.297153:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204632.297158:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000400:00000200:2.0:1788204632.297158:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000100:00000001:0.0:1788204632.297160:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204632.297161:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00100000:0.0:1788204632.297166:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204632.297166:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204632.297173:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204632.297174:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204632.297177:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204632.297184:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297187:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204632.297191:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204632.297193:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204632.297195:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6097) 00000100:00000001:2.0:1788204632.297201:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297203:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204632.297207:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297210:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204632.297213:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300763649 00000100:00000010:2.0:1788204632.297217:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1b40. 00000100:00000001:2.0:1788204632.297222:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204632.297226:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300763649 00000100:00000010:2.0:1788204632.297228:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1840. 00000100:00000001:2.0:1788204632.297230:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204632.297232:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204632.297234:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297235:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204632.297238:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204632/real 1788204632] req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204632.297243:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204632.297244:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204632.297245:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204632.297247:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204632.297248:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204632.297249:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204632.297251:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204632.297256:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297259:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204632.297261:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297262:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297263:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204632.297265:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297269:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204632.297272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297275:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204632.297277:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297278:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297278:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297280:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204632.297281:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204632.297283:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204632.297285:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204632.297286:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204632.297288:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204632.297293:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205881984:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204632.297295:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297296:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.297298:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205881984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297302:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204632.297304:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204632.297307:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204632.297309:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204632.297311:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204632.297313:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204632.297315:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204632.297317:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:2.0:1788204632.297320:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204632.297322:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204632.297323:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204632.297326:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204632.297327:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297330:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297336:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204632.297337:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297339:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204632.297341:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204632.297342:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204632.297346:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297353:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204632.297354:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297356:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204632.297358:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204632.297360:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204632.297387:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882112:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204632.297392:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204632.297393:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204632.297395:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297397:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204632.297400:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204632.297402:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297405:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204632.297407:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204632.297410:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204632.297413:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882112, portal 10 00000100:00000001:2.0:1788204632.297415:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204632.297416:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204632.297422:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297429:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204632.297432:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204632.297434:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204632.297436:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205882112, offset 0 00000400:00000200:2.0:1788204632.297440:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297443:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297446:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204632.297449:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204632.297452:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204632.297454:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204632.297456:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297459:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58747:58747:256:4294967295] 192.168.201.151@tcp LPNI seq info [58727:58727:8:4294967295] 00000400:00000200:2.0:1788204632.297463:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297470:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204632.297474:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297477:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204632.297481:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204632.297484:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204632.297488:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204632.297491:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204632.297494:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297496:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204632.297499:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204632.297502:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204632.297505:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204632.297507:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204632.297509:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204632.297512:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297520:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.297524:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297536:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297538:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204632.297540:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204632.297544:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204632.297547:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204632.297549:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300759707 00000100:00000001:2.0:1788204632.297552:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.297554:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300759707 00000100:00000001:2.0:1788204632.297556:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.297557:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204632.297559:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297561:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204632.297565:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204632/real 1788204632] req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204632.297572:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204632.297573:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204632.297579:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204632.297582:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204632.297584:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204632.297586:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204632.297588:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204632.297593:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297597:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204632.297598:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297599:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297600:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204632.297602:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297606:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204632.297609:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297612:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204632.297614:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297614:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297615:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297616:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204632.297617:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204632.297619:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204632.297621:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204632.297629:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204632.297633:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204632.297644:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882112:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204632.297647:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204632.297650:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297656:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204632.297658:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297660:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204632.297661:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204632.297663:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204632.297666:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882240:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204632.297668:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204632.297670:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204632.297671:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297672:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204632.297674:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204632.297676:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297678:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204632.297680:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204632.297682:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204632.297684:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882240, portal 4 00000100:00000001:2.0:1788204632.297685:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204632.297686:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204632.297690:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297693:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204632.297695:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204632.297696:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204632.297697:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205882240, offset 0 00000400:00000200:2.0:1788204632.297699:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297702:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297704:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204632.297706:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204632.297707:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204632.297709:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204632.297710:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297712:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58748:58748:256:4294967295] 192.168.201.151@tcp LPNI seq info [58728:58728:8:4294967295] 00000400:00000200:2.0:1788204632.297714:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297718:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204632.297721:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297722:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204632.297724:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204632.297726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204632.297729:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204632.297730:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204632.297733:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297734:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204632.297737:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204632.297740:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204632.297742:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204632.297744:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204632.297746:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204632.297749:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297756:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.297759:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297764:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297766:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204632.297769:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204632.297771:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204632.297774:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204632.297776:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300759593 00000100:00000001:2.0:1788204632.297778:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.297779:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300759593 00000100:00000001:2.0:1788204632.297781:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.297783:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204632.297785:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.297786:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204632.297790:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204632/real 1788204632] req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204632.297796:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204632.297797:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204632.297799:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204632.297801:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204632.297802:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204632.297805:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204632.297808:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204632.297816:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297822:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204632.297823:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297825:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297826:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204632.297829:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297834:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204632.297836:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297840:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204632.297841:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297842:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.297842:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.297844:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204632.297845:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204632.297846:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204632.297848:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204632.297849:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204632.297851:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204632.297855:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882240:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204632.297857:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204632.297860:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297866:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204632.297867:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297869:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204632.297871:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204632.297872:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204632.297875:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204632.297878:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204632.297879:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204632.297892:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204632.297894:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204632.297897:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204632.297899:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204632.297902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204632.297904:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204632.297907:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204632.297910:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882368, portal 4 00000100:00000001:2.0:1788204632.297912:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204632.297913:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204632.297919:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.297925:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204632.297926:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204632.297928:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204632.297929:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205882368, offset 0 00000400:00000200:2.0:1788204632.297932:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297934:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297944:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204632.297949:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204632.297952:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204632.297954:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204632.297956:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204632.297959:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58749:58749:256:4294967295] 192.168.201.151@tcp LPNI seq info [58729:58729:8:4294967295] 00000400:00000200:2.0:1788204632.297964:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204632.297970:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204632.297973:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297976:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204632.297979:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204632.297981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204632.297985:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204632.297987:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204632.297991:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204632.297992:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204632.297995:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204632.297998:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204632.298000:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204632.298002:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204632.298005:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204632.298008:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298013:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.298016:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298021:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.298023:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204632.298025:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204632.298028:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204632.298031:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204632.298032:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300759599 00000100:00000001:2.0:1788204632.298034:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.298036:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300759599 00000100:00000001:2.0:1788204632.298038:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300763649 00000100:00000001:2.0:1788204632.298040:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204632.298042:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.298044:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204632.298047:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204632/real 1788204632] req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204632.298053:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204632.298054:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204632.298055:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204632.298057:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204632.298058:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204632.298060:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204632.298063:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204632.298069:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298075:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204632.298077:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298078:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.298080:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204632.298083:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298088:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204632.298090:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298094:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204632.298095:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298096:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204632.298097:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298098:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204632.298099:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204632.298100:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204632.298102:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204632.298104:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204632.298106:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204632.298110:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204632.298112:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298113:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.298115:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205882112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298118:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204632.298119:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204632.298120:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204632.298122:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204632.298123:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204632.298124:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204632.298125:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204632.298126:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:2.0:1788204632.298128:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204632.298129:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204632.298130:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204632.298132:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204632.298132:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298133:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.298135:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205882240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298139:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204632.298139:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204632.298140:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204632.298141:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204632.298142:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204632.298142:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204632.298143:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204632.298144:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:2.0:1788204632.298146:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204632.298147:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204632.298148:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204632.298149:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204632.298150:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298150:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204632.298152:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205882368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204632.298155:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204632.298155:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204632.298156:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204632.298157:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204632.298158:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204632.298158:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204632.298159:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204632.298160:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:2.0:1788204632.298161:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204632.298162:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204632.298162:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204632.298163:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204632.298164:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298165:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204632.298167:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204632.298168:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.298169:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204632.298170:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204632.488468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204632.488474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204632.488480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204632.488481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204632.488484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204632.488486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204632.488490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204632.488491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204632.488542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204632.488545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204632.488548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204633.256479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204633.256490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204633.256493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204633.256499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204633.256504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204633.256508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204633.256510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204633.256512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204633.256514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204633.320465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204633.320472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204633.320474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204633.320479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204633.320483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204633.320487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204633.320491:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204633.320494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204633.320497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204633.320529:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204633.320536:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204633.320539:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204633.320543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204633.320546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204633.320550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204633.320552:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204633.320555:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204633.320557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204633.512531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204633.512538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204633.512543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204633.512544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204633.512546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204633.512547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204633.512549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204633.512550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204633.512566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204633.512568:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204633.512571:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204634.280492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204634.280502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204634.280504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204634.280511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204634.280522:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204634.280527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204634.280530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204634.280533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204634.280536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204634.344535:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204634.344538:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204634.344547:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204634.344548:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204634.344554:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204634.344554:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204634.344561:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204634.344563:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204634.344566:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204634.344570:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204634.344572:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204634.344574:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204634.344577:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204634.344577:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204634.344579:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204634.344585:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204634.344589:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204634.344594:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204634.536463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204634.536470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204634.536476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204634.536478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204634.536481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204634.536483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204634.536487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204634.536488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204634.536544:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204634.536547:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204634.536549:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204635.304436:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204635.304443:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204635.304445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204635.304450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204635.304453:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204635.304463:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204635.304469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204635.304471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204635.304474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204635.368439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204635.368445:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204635.368447:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204635.368451:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204635.368454:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204635.368457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204635.368459:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204635.368461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204635.368461:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204635.368465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204635.368466:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204635.368468:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204635.368471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204635.368473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204635.368474:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204635.368476:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204635.368477:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204635.368479:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204635.560514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204635.560521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204635.560527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204635.560528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204635.560531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204635.560532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204635.560535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204635.560536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204635.560552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204635.560555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204635.560558:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204636.328493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204636.328500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204636.328502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204636.328507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204636.328510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204636.328514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204636.328517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204636.328519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204636.328522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204636.392505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204636.392505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204636.392516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204636.392518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204636.392519:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204636.392522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204636.392525:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204636.392529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204636.392530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204636.392533:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204636.392540:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204636.392540:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204636.392546:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204636.392549:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204636.392550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204636.392560:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204636.392566:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204636.392568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204636.584463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204636.584467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204636.584471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204636.584472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204636.584474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204636.584475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204636.584477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204636.584478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204636.584545:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204636.584549:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204636.584552:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204636.904446:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300763649, at 4300764160 00000100:00000040:1.0:1788204636.904446:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300763649, at 4300764160 00000100:00000010:0.0:1788204636.904460:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000010:1.0:1788204636.904462:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000001:1.0:1788204637.352432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204637.352438:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204637.352440:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204637.352443:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204637.352446:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204637.352449:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204637.352450:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204637.352453:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204637.352455:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204637.416458:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204637.416458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204637.416463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204637.416466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204637.416467:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204637.416472:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204637.416474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204637.416477:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204637.416479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204637.416482:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204637.416484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204637.416488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204637.416489:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6092 00000100:00000001:1.0:1788204637.416491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204637.416498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204637.416503:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204637.416514:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204637.416519:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204637.416526:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416529:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.416531:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416532:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204637.416535:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204637.416536:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204637.416538:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204637.416541:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204637.416542:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204637.416544:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204637.416546:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416547:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.416548:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416549:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204637.416551:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204637.416553:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416555:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204637.416556:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204637.416559:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204637.416561:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.416562:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204637.416564:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204637.416572:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204637.416575:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204637.416578:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204637.416579:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.416581:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204637.416583:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204637.416586:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204637.416593:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204637.416598:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204637.416605:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204637.416616:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204637.416619:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204637.416622:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204637.416639:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204637.416641:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204637.416643:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204637.416644:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204637.416645:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204637.416647:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204637.416649:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6092 00000400:00000200:0.0:1788204637.416651:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204637.416654:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204637.416656:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204637.416658:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416659:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.416662:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416664:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204637.416665:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204637.416666:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204637.416667:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204637.416668:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204637.416670:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204637.416671:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204637.416672:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416673:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.416674:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416675:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204637.416677:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204637.416679:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416680:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204637.416683:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204637.416683:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788204637.416690:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204637.416692:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788204637.416696:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.416698:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204637.416705:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204637.416710:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1788204637.416713:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204637.416720:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.416728:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.416731:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204637.416736:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204637.416738:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 02000000:00000001:2.0:1788204637.416740:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204637.416746:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788204637.416747:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204637.416750:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204637.416759:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788204637.416760:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205882496:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204637.416764:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.416766:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204637.416772:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204637.416775:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204637.416775:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204637.416780:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204637.416781:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.416784:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204637.416789:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204637.416808:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204637.416816:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1788204637.416821:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204637.416829:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.416834:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204637.416839:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.416847:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204637.416848:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204637.416857:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00000001:0.0:1788204637.416866:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204637.416871:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1788204637.416872:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00080000:0.0:1788204637.416882:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204637.416884:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1788204637.416885:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882496, portal 25 00000100:00080000:0.0:1788204637.416886:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204637.416890:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204637.416893:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204637.416894:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204637.416900:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6092 00000400:00000200:0.0:1788204637.416906:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788204637.416912:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788204637.416914:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204637.416920:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204637.416928:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416937:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.416945:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204637.416947:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204637.416954:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:2.0:1788204637.416960:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788204637.416962:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:2.0:1788204637.416964:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000020:00000040:0.0:1788204637.416965:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204637.416972:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:2.0:1788204637.416972:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205882496, offset 0 00000020:00000040:0.0:1788204637.416977:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204637.416982:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204637.416982:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788204637.416989:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204637.416993:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.416995:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204637.417000:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204637.417005:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.417008:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204637.417010:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204637.417012:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:0.0:1788204637.417019:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204637.417024:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417025:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1788204637.417032:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204637.417036:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204637.417037:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:0.0:1788204637.417047:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.417051:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204637.417052:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000008:00100000:0.0:1788204637.417055:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204637.417058:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417065:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.417066:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204637.417075:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204637.417079:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204637.417081:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204637.417082:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.417083:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204637.417084:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204637.417085:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417090:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58750:58750:256:4294967295] 192.168.201.151@tcp LPNI seq info [58730:58730:8:4294967295] 02000000:00000010:0.0:1788204637.417091:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204637.417100:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417106:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:0.0:1788204637.417108:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204637.417115:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204637.417122:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204637.417133:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204637.417138:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204637.417146:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204637.417148:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204637.417165:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204637.417168:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204637.417170:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204637.417177:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.0:1788204637.417177:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204637.417184:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6092 00000800:00000010:2.0:1788204637.417187:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000400:00000200:0.0:1788204637.417188:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204637.417198:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204637.417201:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1788204637.417205:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204637.417211:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204637.417213:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1788204637.417214:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.417221:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204637.417227:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:2.1:1788204637.417228:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:0.0:1788204637.417231:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204637.417237:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000200:2.1:1788204637.417237:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000020:00000001:0.0:1788204637.417243:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204637.417247:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000100:2.0:1788204637.417248:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788204637.417251:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000010:2.0:1788204637.417252:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000020:00000001:0.0:1788204637.417260:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204637.417266:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417267:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204637.417269:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204637.417277:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417278:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788204637.417287:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204637.417292:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000020:00000040:0.0:1788204637.417293:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000010:2.0:1788204637.417299:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00080000:0.0:1788204637.417308:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204637.417310:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204637.417317:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204637.417322:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204637.417325:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204637.417332:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204637.417336:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788204637.417343:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204637.417343:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204637.417354:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204637.417356:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417359:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1788204637.417360:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:2.0:1788204637.417388:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417393:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204637.417397:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300768770 00000100:00000010:2.0:1788204637.417402:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1900. 00000008:00000001:0.0:1788204637.417403:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204637.417408:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417412:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000010:0.0:1788204637.417414:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204637.417419:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204637.417423:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204637.417425:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417425:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300768770 00000100:00000001:0.0:1788204637.417433:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:2.0:1788204637.417435:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a80. 02000000:00000001:0.0:1788204637.417436:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204637.417439:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417444:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000010:0.0:1788204637.417447:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:2.0:1788204637.417451:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204637.417453:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417456:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417461:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204637.417463:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788204637.417472:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204637/real 1788204637] req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204637.417478:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204637.417482:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204637.417491:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204637.417495:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788204637.417501:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204637.417501:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204637.417506:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000100:00000040:0.0:1788204637.417509:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6102) 00000400:00000200:2.0:1788204637.417512:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204637.417516:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204637.417520:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204637.417525:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204637.417534:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417541:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204637.417543:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417545:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.417547:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204637.417551:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417558:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204637.417562:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417569:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204637.417572:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417573:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.417574:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417577:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204637.417579:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204637.417583:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204637.417587:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204637.417589:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204637.417594:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204637.417599:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205882496:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204637.417604:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417606:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.417608:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205882496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417612:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204637.417614:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204637.417616:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204637.417619:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204637.417620:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204637.417622:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204637.417623:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204637.417626:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:2.0:1788204637.417629:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204637.417630:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204637.417632:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204637.417635:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204637.417637:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417639:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417647:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204637.417649:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417652:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204637.417653:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204637.417655:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204637.417659:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417667:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204637.417668:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.417670:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204637.417672:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204637.417675:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204637.417678:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882624:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204637.417681:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204637.417683:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204637.417685:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.417687:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204637.417690:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204637.417693:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417697:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204637.417699:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204637.417703:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204637.417706:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882624, portal 10 00000100:00000001:2.0:1788204637.417708:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204637.417709:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204637.417717:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417723:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204637.417726:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204637.417728:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204637.417731:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205882624, offset 0 00000400:00000200:2.0:1788204637.417735:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417738:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.417742:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204637.417745:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204637.417748:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204637.417750:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204637.417752:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417755:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58751:58751:256:4294967295] 192.168.201.151@tcp LPNI seq info [58731:58731:8:4294967295] 00000400:00000200:2.0:1788204637.417759:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.417765:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204637.417769:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.417772:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204637.417776:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204637.417779:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204637.417783:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204637.417816:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204637.417821:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.417823:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204637.417827:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204637.417831:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204637.417834:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204637.417836:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204637.417839:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204637.417843:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417848:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.417850:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417854:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417855:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204637.417858:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204637.417861:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204637.417864:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204637.417865:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300764827 00000100:00000001:2.0:1788204637.417867:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.417869:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300764827 00000100:00000001:2.0:1788204637.417870:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.417872:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204637.417873:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.417874:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204637.417876:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204637/real 1788204637] req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204637.417880:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204637.417881:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204637.417882:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204637.417883:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204637.417884:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204637.417885:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204637.417888:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204637.417893:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417897:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204637.417898:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417899:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.417899:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204637.417901:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417905:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204637.417907:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417910:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204637.417912:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417912:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.417913:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.417914:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204637.417916:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204637.417917:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204637.417919:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204637.417921:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204637.417923:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204637.417928:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882624:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204637.417930:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204637.417932:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417940:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204637.417941:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.417942:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204637.417944:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204637.417945:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204637.417946:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882752:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204637.417948:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204637.417949:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204637.417950:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.417951:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204637.417954:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204637.417955:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.417957:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204637.417959:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204637.417961:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204637.417963:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882752, portal 4 00000100:00000001:2.0:1788204637.417964:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204637.417965:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204637.417969:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.417973:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204637.417974:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204637.417976:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204637.417977:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205882752, offset 0 00000400:00000200:2.0:1788204637.417979:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417981:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.417983:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204637.417985:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204637.417987:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204637.417988:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204637.417990:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204637.417992:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58752:58752:256:4294967295] 192.168.201.151@tcp LPNI seq info [58732:58732:8:4294967295] 00000400:00000200:2.0:1788204637.417995:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.418000:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204637.418003:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.418005:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204637.418007:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204637.418009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204637.418011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204637.418013:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204637.418015:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.418016:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204637.418018:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204637.418021:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204637.418022:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204637.418024:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204637.418025:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204637.418027:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418032:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.418034:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418037:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.418038:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204637.418040:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204637.418043:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204637.418045:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204637.418047:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300764713 00000100:00000001:2.0:1788204637.418049:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.418052:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300764713 00000100:00000001:2.0:1788204637.418054:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.418055:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204637.418058:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.418060:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204637.418063:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204637/real 1788204637] req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204637.418071:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204637.418072:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204637.418074:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204637.418076:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204637.418078:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204637.418080:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204637.418083:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204637.418091:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418095:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204637.418096:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418098:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.418099:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204637.418101:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418105:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204637.418107:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418111:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204637.418112:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418113:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.418113:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418114:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204637.418116:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204637.418118:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204637.418121:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204637.418123:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204637.418127:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204637.418135:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882752:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204637.418139:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204637.418142:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418149:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204637.418150:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.418153:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204637.418155:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204637.418157:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204637.418160:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204637.418162:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204637.418164:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204637.418165:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204637.418166:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204637.418169:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204637.418170:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204637.418172:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204637.418174:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204637.418176:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204637.418178:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205882880, portal 4 00000100:00000001:2.0:1788204637.418179:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204637.418180:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204637.418184:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418188:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204637.418190:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204637.418191:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204637.418193:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205882880, offset 0 00000400:00000200:2.0:1788204637.418196:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204637.418198:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.418200:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204637.418202:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204637.418204:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204637.418206:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204637.418207:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204637.418209:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58753:58753:256:4294967295] 192.168.201.151@tcp LPNI seq info [58733:58733:8:4294967295] 00000400:00000200:2.0:1788204637.418212:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204637.418216:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204637.418218:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.418220:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000800:00000200:2.0:1788204637.418223:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204637.418225:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204637.418227:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204637.418229:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204637.418231:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204637.418232:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:2.0:1788204637.418234:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204637.418236:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204637.418238:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:2.0:1788204637.418239:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:2.0:1788204637.418240:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204637.418242:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418247:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.418249:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418253:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.418254:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204637.418256:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204637.418258:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204637.418260:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204637.418261:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300764719 00000100:00000001:2.0:1788204637.418263:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.418264:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300764719 00000100:00000001:2.0:1788204637.418266:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300768770 00000100:00000001:2.0:1788204637.418266:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204637.418268:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.418269:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204637.418270:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204637/real 1788204637] req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204637.418274:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204637.418275:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204637.418276:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204637.418278:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204637.418279:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204637.418280:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204637.418281:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204637.418286:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418290:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204637.418291:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418292:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.418293:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204637.418295:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418298:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204637.418300:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418305:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204637.418306:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418307:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204637.418308:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418309:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204637.418311:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204637.418312:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204637.418314:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204637.418315:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204637.418317:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204637.418321:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205882880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204637.418322:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418324:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.418325:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205882624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418328:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204637.418329:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204637.418330:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204637.418332:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204637.418332:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204637.418333:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204637.418334:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204637.418335:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:2.0:1788204637.418337:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204637.418338:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204637.418339:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204637.418340:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204637.418341:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418343:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.418344:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205882752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418348:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204637.418349:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204637.418350:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204637.418351:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204637.418352:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204637.418353:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204637.418354:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204637.418354:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:2.0:1788204637.418356:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204637.418356:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204637.418357:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204637.418358:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204637.418359:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418360:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204637.418361:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205882880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204637.418381:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204637.418381:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204637.418382:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204637.418383:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204637.418384:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204637.418385:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204637.418386:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204637.418386:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:2.0:1788204637.418388:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204637.418389:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204637.418390:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204637.418391:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204637.418391:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418392:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204637.418395:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204637.418395:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.418396:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204637.418398:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204637.608524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204637.608531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204637.608537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204637.608539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204637.608542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204637.608543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204637.608546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204637.608547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204637.608566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204637.608569:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204637.608575:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204638.376463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204638.376470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204638.376472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204638.376477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204638.376480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204638.376484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204638.376486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204638.376488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204638.376490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204638.440467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204638.440476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204638.440480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204638.440487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204638.440493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204638.440498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204638.440512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204638.440515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204638.440517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204638.632473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204638.632479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204638.632485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204638.632486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204638.632489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204638.632491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204638.632495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204638.632496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204638.632567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204638.632571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204638.632574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204639.400507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204639.400517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204639.400520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204639.400527:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204639.400532:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204639.400537:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204639.400540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204639.400544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204639.400547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204639.464496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204639.464505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204639.464507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204639.464512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204639.464515:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204639.464519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204639.464522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204639.464523:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204639.464524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204639.464530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204639.464531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204639.464534:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204639.464539:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204639.464542:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204639.464544:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204639.464546:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204639.464548:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204639.464550:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204639.656548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204639.656558:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204639.656561:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204639.656565:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656568:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204639.656571:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656573:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656575:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656579:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204639.656580:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204639.656583:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204639.656585:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204639.656604:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204639.656607:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204639.656610:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204640.424461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204640.424468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204640.424470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204640.424475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204640.424479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204640.424482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204640.424484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204640.424487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204640.424490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204640.488498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204640.488505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204640.488507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204640.488512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204640.488516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204640.488520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204640.488525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204640.488528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204640.488531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204640.488546:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204640.488553:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204640.488555:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204640.488561:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204640.488564:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204640.488577:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204640.488583:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204640.488587:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204640.488591:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204640.680470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204640.680475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204640.680480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204640.680482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204640.680485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204640.680486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204640.680489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204640.680490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204640.680562:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204640.680567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204640.680571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204641.448444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204641.448449:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204641.448451:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204641.448454:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204641.448457:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204641.448459:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204641.448461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204641.448463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204641.448465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204641.512463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204641.512463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204641.512470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204641.512470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204641.512472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204641.512473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204641.512478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204641.512479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204641.512483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204641.512484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204641.512487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204641.512487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204641.512492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204641.512495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204641.512496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204641.512498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204641.512500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204641.512502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204641.704477:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204641.704479:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204641.704482:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704485:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204641.704486:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704487:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704489:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204641.704492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204641.704495:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204641.704496:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204641.704497:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704498:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204641.704498:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704499:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704500:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204641.704503:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204641.704504:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204641.704504:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704505:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204641.704506:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704507:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204641.704508:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204641.704519:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204641.704520:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204641.704522:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204642.024469:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300768770, at 4300769280 00000100:00000040:1.0:1788204642.024469:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300768770, at 4300769280 00000100:00000010:1.0:1788204642.024484:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000010:0.0:1788204642.024486:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000001:1.0:1788204642.472487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204642.472494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204642.472497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204642.472503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204642.472507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204642.472510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204642.472513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204642.472515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204642.472518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204642.536470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204642.536471:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204642.536477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204642.536481:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204642.536481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.536486:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204642.536487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204642.536491:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204642.536493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204642.536496:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204642.536498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204642.536504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204642.536505:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6097 00000100:00000001:1.0:1788204642.536507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204642.536513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204642.536516:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204642.536525:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204642.536531:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204642.536538:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536542:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.536545:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536547:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204642.536549:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204642.536550:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204642.536552:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204642.536555:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204642.536557:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204642.536558:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204642.536560:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536562:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.536563:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536564:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204642.536566:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204642.536568:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536570:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204642.536572:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204642.536574:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204642.536576:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.536578:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204642.536580:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204642.536589:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204642.536592:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204642.536596:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204642.536598:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.536600:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204642.536602:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204642.536611:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204642.536619:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204642.536625:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204642.536635:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204642.536647:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204642.536652:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204642.536656:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.536679:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204642.536681:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204642.536685:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204642.536686:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204642.536687:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204642.536697:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204642.536699:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6097 00000400:00000200:0.0:1788204642.536703:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204642.536715:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204642.536718:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204642.536721:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536724:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.536726:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536729:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204642.536732:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204642.536733:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:2.0:1788204642.536735:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204642.536736:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204642.536741:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204642.536742:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204642.536744:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204642.536746:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204642.536750:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204642.536756:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:2.0:1788204642.536757:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.536759:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.536764:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536768:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204642.536773:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204642.536776:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204642.536779:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204642.536780:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536785:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.536785:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204642.536789:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204642.536790:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204642.536794:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204642.536797:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788204642.536799:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205883008:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204642.536802:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.536806:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204642.536807:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204642.536810:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788204642.536812:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204642.536813:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.536816:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788204642.536819:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204642.536823:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:0.0:1788204642.536824:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.536827:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204642.536831:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000200:2.0:1788204642.536832:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:0.0:1788204642.536837:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:2.0:1788204642.536839:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 02000000:00000001:0.0:1788204642.536842:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204642.536845:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204642.536847:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204642.536848:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204642.536851:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204642.536855:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204642.536856:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883008, portal 25 02000000:00000010:0.0:1788204642.536860:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:2.0:1788204642.536860:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204642.536863:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000001:0.0:1788204642.536866:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204642.536872:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204642.536876:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204642.536893:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204642.536897:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204642.536898:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204642.536902:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204642.536903:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85330. 00000100:00000200:2.0:1788204642.536908:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205883008, offset 0 00000100:00000040:0.0:1788204642.536910:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204642.536915:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204642.536926:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.536934:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204642.536937:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204642.536938:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1788204642.536942:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204642.536944:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:0.0:1788204642.536950:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1788204642.536950:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788204642.536955:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1788204642.536955:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:0.0:1788204642.536960:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204642.536961:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58754:58754:256:4294967295] 192.168.201.151@tcp LPNI seq info [58734:58734:8:4294967295] 00000400:00000200:2.0:1788204642.536967:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204642.536974:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204642.536982:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6097 00000400:00000200:2.0:1788204642.536983:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:0.0:1788204642.536992:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:2.0:1788204642.536993:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204642.537000:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000010:2.0:1788204642.537001:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4900. 00000100:00080000:0.0:1788204642.537005:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204642.537010:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204642.537013:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204642.537015:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.537020:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204642.537025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204642.537026:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204642.537032:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204642.537033:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204642.537035:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204642.537037:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204642.537039:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204642.537041:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204642.537046:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204642.537046:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.537052:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204642.537055:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204642.537059:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300773889 00000100:00000040:0.0:1788204642.537062:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:2.0:1788204642.537064:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1600. 00000100:00000001:0.0:1788204642.537066:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204642.537072:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000001:2.0:1788204642.537072:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788204642.537078:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204642.537079:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300773889 00000100:00000001:0.0:1788204642.537085:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204642.537085:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1660. 00000100:00000001:2.0:1788204642.537089:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788204642.537090:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204642.537093:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204642.537095:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204642.537096:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204642.537103:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537103:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537107:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204642.537108:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204642.537112:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:2.0:1788204642.537113:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000008:00000001:0.0:1788204642.537115:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.537119:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537120:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000010:0.0:1788204642.537124:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204642.537124:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204642.537126:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204642.537129:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1788204642.537131:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000040:2.0:1788204642.537135:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204642.537137:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204642.537140:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.537145:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204642.537146:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788204642.537150:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204642.537150:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204642.537155:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537155:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204642.537157:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204642.537160:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204642.537162:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00100000:2.0:1788204642.537162:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883136:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204642.537167:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204642.537169:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537169:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204642.537172:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537174:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204642.537178:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 00000100:00100000:0.0:1788204642.537179:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204642.537181:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204642.537189:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204642.537193:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aacb8. 00000400:00000010:2.0:1788204642.537197:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000001:0.0:1788204642.537198:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1788204642.537201:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883136, portal 10 00000100:00000001:0.0:1788204642.537203:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537204:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204642.537208:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:0.0:1788204642.537212:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204642.537216:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204642.537239:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204642.537243:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204642.537243:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204642.537247:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1788204642.537248:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85220. 00000100:00080000:0.0:1788204642.537252:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:2.0:1788204642.537254:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205883136, offset 0 00000100:00000001:0.0:1788204642.537255:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204642.537260:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204642.537264:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204642.537266:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204642.537272:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204642.537273:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6097 00000400:00000200:2.0:1788204642.537281:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204642.537282:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.537288:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204642.537291:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204642.537295:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:0.0:1788204642.537296:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204642.537301:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204642.537304:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537308:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.537310:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.537316:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537317:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58755:58755:255:4294967295] 192.168.201.151@tcp LPNI seq info [58735:58735:7:4294967295] 00000100:00000001:0.0:1788204642.537320:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204642.537326:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1788204642.537327:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:0.0:1788204642.537329:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204642.537333:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204642.537337:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:2.0:1788204642.537338:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:0.0:1788204642.537342:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204642.537349:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:2.0:1788204642.537349:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204642.537358:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000010:2.0:1788204642.537358:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00000001:0.0:1788204642.537383:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204642.537385:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204642.537388:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204642.537391:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204642.537394:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204642.537397:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204642.537401:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204642.537402:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204642.537406:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1788204642.537411:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204642.537412:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.537418:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204642.537418:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204642.537422:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000040:2.0:1788204642.537425:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000008:00000001:0.0:1788204642.537428:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204642.537432:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204642.537433:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537437:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300769947 00000100:00000010:0.0:1788204642.537440:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204642.537442:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300773889 00000020:00000040:0.0:1788204642.537446:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204642.537447:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300769947 00000100:00000001:2.0:1788204642.537452:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300773889 02000000:00000001:0.0:1788204642.537453:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204642.537454:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204642.537456:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537458:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.537461:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204642.537463:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204642.537464:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204642.537467:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204642.537472:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204642.537474:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1788204642.537475:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537477:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204642.537480:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000010:0.0:1788204642.537482:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:2.0:1788204642.537482:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204642.537488:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204642.537489:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204642.537498:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204642.537508:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204642.537510:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537510:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204642.537513:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204642.537517:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204642.537519:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204642.537522:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204642.537527:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204642.537535:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883264:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788204642.537536:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537542:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788204642.537544:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6107) 02000000:00000001:2.0:1788204642.537546:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204642.537550:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537552:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204642.537556:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204642.537558:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204642.537563:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204642.537566:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa1b8. 00000400:00000010:2.0:1788204642.537574:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85088. 00000100:00000200:2.0:1788204642.537577:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883264, portal 4 00000100:00000001:2.0:1788204642.537579:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204642.537580:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204642.537586:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204642.537592:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204642.537594:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204642.537596:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85880. 00000100:00000200:2.0:1788204642.537598:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205883264, offset 0 00000400:00000200:2.0:1788204642.537602:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.537609:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204642.537612:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204642.537614:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204642.537616:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204642.537618:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537621:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58756:58756:254:4294967295] 192.168.201.151@tcp LPNI seq info [58736:58736:6:4294967295] 00000400:00000200:2.0:1788204642.537639:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.537645:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204642.537649:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204642.537652:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204642.537656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204642.537659:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204642.537662:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204642.537665:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204642.537668:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204642.537669:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300769833 00000100:00000001:2.0:1788204642.537671:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300773889 00000100:00000001:2.0:1788204642.537673:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300769833 00000100:00000001:2.0:1788204642.537675:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300773889 00000100:00000001:2.0:1788204642.537677:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204642.537679:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537681:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537682:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204642.537685:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204642.537687:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204642.537689:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204642.537690:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204642.537691:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204642.537694:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204642.537700:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204642.537702:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537712:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204642.537714:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204642.537715:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204642.537718:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204642.537734:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204642.537736:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204642.537737:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204642.537739:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204642.537743:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204642.537745:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204642.537749:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204642.537751:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa630. 00000400:00000010:2.0:1788204642.537754:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85908. 00000100:00000200:2.0:1788204642.537757:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883392, portal 4 00000100:00000001:2.0:1788204642.537759:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204642.537760:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204642.537765:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204642.537771:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204642.537773:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204642.537775:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85bb0. 00000100:00000200:2.0:1788204642.537778:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205883392, offset 0 00000400:00000200:2.0:1788204642.537782:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537786:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.537789:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204642.537791:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204642.537793:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204642.537795:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204642.537797:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204642.537799:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58757:58757:253:4294967295] 192.168.201.151@tcp LPNI seq info [58737:58737:5:4294967295] 00000400:00000200:2.0:1788204642.537802:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204642.537807:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204642.537811:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204642.537814:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4c00. 00000800:00000200:2.0:1788204642.537817:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204642.537821:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204642.537823:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204642.537825:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204642.537829:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204642.537830:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300769839 00000100:00000001:2.0:1788204642.537832:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300773889 00000100:00000001:2.0:1788204642.537835:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300769839 00000100:00000001:2.0:1788204642.537836:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300773889 00000100:00000001:2.0:1788204642.537838:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204642.537840:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537842:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537844:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204642.537847:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204642.537848:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204642.537850:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204642.537851:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204642.537853:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.537855:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204642.728461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204642.728468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204642.728474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204642.728475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204642.728479:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204642.728480:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204642.728484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204642.728485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204642.728550:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204642.728554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204642.728557:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204643.496490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204643.496498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204643.496501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204643.496506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204643.496510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204643.496513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204643.496516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204643.496518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204643.496521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204643.560458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204643.560466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204643.560468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204643.560473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204643.560479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204643.560483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204643.560488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204643.560491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204643.560495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204643.752484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204643.752491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204643.752497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204643.752498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204643.752501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204643.752502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204643.752505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204643.752506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204643.752523:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204643.752526:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204643.752530:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204644.520431:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204644.520436:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204644.520438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204644.520441:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204644.520444:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204644.520447:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204644.520448:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204644.520450:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204644.520452:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204644.584461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204644.584468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204644.584470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204644.584475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204644.584478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204644.584482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204644.584485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204644.584488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204644.584491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204644.776495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204644.776500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204644.776505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204644.776506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204644.776510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204644.776512:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204644.776515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204644.776516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204644.776571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204644.776574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204644.776577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204645.544458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204645.544462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204645.544464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204645.544468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204645.544471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204645.544474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204645.544476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204645.544477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204645.544479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204645.608466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204645.608472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204645.608474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204645.608478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204645.608480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204645.608483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204645.608485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204645.608487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204645.608490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204645.608747:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:0.0:1788204645.608800:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204645.608812:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204645.608816:0:2334:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:0.0:1788204645.608822:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204645.608826:0:2334:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788204645.608832:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204645.608837:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4900. 00000400:00000200:0.0:1788204645.608843:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204645.608847:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204645.608855:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85330 00000400:00000010:0.0:1788204645.608858:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85330. 00000100:00000001:0.0:1788204645.608864:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204645.608874:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.608905:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204645.608911:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.608918:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204645.608920:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204645.608931:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204645.608934:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:0.0:1788204645.608949:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204645.608952:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204645.608955:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85220 00000400:00000010:0.0:1788204645.608957:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85220. 00000100:00000001:0.0:1788204645.608959:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204645.608964:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.608971:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204645.608974:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.608980:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204645.608982:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204645.608986:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204645.608988:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:0.0:1788204645.608991:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204645.608994:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204645.608996:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85880 00000400:00000010:0.0:1788204645.608997:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85880. 00000100:00000001:0.0:1788204645.609000:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204645.609004:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.609011:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204645.609014:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.609019:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204645.609021:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204645.609025:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204645.609027:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4c00. 00000400:00000200:0.0:1788204645.609030:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204645.609033:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204645.609036:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85bb0 00000400:00000010:0.0:1788204645.609038:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85bb0. 00000100:00000001:0.0:1788204645.609039:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204645.609044:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.609051:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204645.609053:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204645.609059:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204645.609061:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204645.609171:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204645.609177:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204645.609180:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204645.609190:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204642/real 1788204645] req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204645.609204:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204645.609207:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204645.609211:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa630. 00000400:00000200:2.0:1788204645.609215:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85908 00000400:00000010:2.0:1788204645.609217:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85908. 00000100:00000001:2.0:1788204645.609220:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204645.609224:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204645.609235:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609241:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204645.609244:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609247:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609250:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204645.609254:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609261:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204645.609267:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609275:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204645.609279:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609280:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609282:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609285:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204645.609289:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204645.609293:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204645.609298:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204645.609301:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204645.609306:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204645.609316:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204645.609321:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204645.609325:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204642/real 1788204645] req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204645.609333:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204645.609335:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204645.609337:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa1b8. 00000400:00000200:2.0:1788204645.609340:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85088 00000400:00000010:2.0:1788204645.609342:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85088. 00000100:00000001:2.0:1788204645.609344:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204645.609346:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204645.609354:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609360:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204645.609361:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609385:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609387:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204645.609391:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609398:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204645.609401:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609408:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204645.609410:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609412:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609413:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609415:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204645.609417:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204645.609420:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204645.609423:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204645.609425:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204645.609429:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204645.609437:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883264:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204645.609441:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204645.609445:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204642/real 1788204645] req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204645.609453:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204645.609454:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204645.609456:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aacb8. 00000400:00000200:2.0:1788204645.609458:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204645.609460:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204645.609462:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204645.609465:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204645.609472:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609478:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204645.609480:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609482:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609483:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204645.609486:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609492:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204645.609496:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609515:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204645.609517:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609519:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609520:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609522:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204645.609525:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204645.609527:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204645.609531:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204645.609534:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204645.609538:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204645.609547:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883136:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204645.609552:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204645.609556:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204642/real 1788204645] req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204645.609564:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204645.609565:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204645.609567:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204645.609571:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204645.609573:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204645.609576:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204645.609580:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204645.609595:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609603:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204645.609605:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609607:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609608:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204645.609613:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609621:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204645.609624:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609631:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204645.609639:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609641:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204645.609642:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609644:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204645.609645:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204645.609648:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204645.609660:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204645.609662:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204645.609666:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204645.609672:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205883008:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204645.609676:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609678:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204645.609681:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205883392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609687:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204645.609691:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204645.609696:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204645.609701:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204645.609705:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204645.609709:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204645.609712:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204645.609715:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:2.0:1788204645.609722:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204645.609724:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204645.609727:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204645.609731:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204645.609732:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609734:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204645.609737:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205883264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609744:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204645.609745:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204645.609746:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204645.609748:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204645.609750:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204645.609751:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204645.609753:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204645.609755:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:2.0:1788204645.609758:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204645.609760:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204645.609761:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204645.609763:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204645.609765:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609767:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204645.609770:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205883136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609776:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204645.609778:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204645.609779:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204645.609782:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204645.609783:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204645.609785:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204645.609786:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204645.609788:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:2.0:1788204645.609789:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204645.609791:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204645.609792:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204645.609795:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204645.609796:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609797:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204645.609800:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205883008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204645.609806:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204645.609807:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204645.609809:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204645.609812:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204645.609813:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204645.609815:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204645.609816:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204645.609818:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:2.0:1788204645.609820:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204645.609822:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204645.609824:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204645.609827:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204645.609829:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609832:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.609843:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204645.609845:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204645.609849:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204645.609851:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204645.800516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204645.800523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204645.800528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204645.800530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204645.800533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204645.800534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204645.800537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204645.800538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204645.800555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204645.800557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204645.800560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204646.568468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204646.568474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204646.568475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204646.568480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204646.568482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204646.568485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204646.568487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204646.568488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204646.568490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204646.632452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204646.632458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204646.632459:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204646.632463:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204646.632465:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204646.632468:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204646.632470:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204646.632472:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204646.632474:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204646.824463:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204646.824469:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204646.824473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204646.824474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204646.824477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204646.824478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204646.824480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204646.824481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204646.824528:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204646.824532:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204646.824536:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204647.144468:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300773889, at 4300774400 00000100:00000010:0.0:1788204647.144482:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000040:1.0:1788204647.144584:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300773889, at 4300774400 00000100:00000010:1.0:1788204647.144590:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:1.0:1788204647.592429:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204647.592435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204647.592436:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204647.592439:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204647.592442:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204647.592451:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204647.592452:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204647.592454:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204647.592458:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204647.656436:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204647.656436:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204647.656441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204647.656443:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204647.656444:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.656447:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204647.656448:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204647.656450:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204647.656452:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204647.656455:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204647.656457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204647.656461:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6102 00000100:00000001:1.0:1788204647.656461:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204647.656465:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204647.656468:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204647.656470:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204647.656476:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204647.656479:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204647.656481:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656484:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.656487:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656492:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204647.656497:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204647.656499:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204647.656504:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204647.656508:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204647.656512:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204647.656515:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204647.656519:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656521:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.656522:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656523:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204647.656525:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204647.656528:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656530:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204647.656532:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204647.656534:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204647.656536:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.656538:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204647.656540:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204647.656545:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204647.656547:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204647.656551:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204647.656552:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.656555:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204647.656558:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204647.656562:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204647.656570:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204647.656576:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204647.656584:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204647.656593:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204647.656598:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204647.656602:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204647.656651:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204647.656654:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204647.656657:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204647.656659:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204647.656660:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204647.656662:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204647.656669:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6102 00000400:00000200:0.0:1788204647.656673:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204647.656676:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204647.656678:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204647.656681:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656683:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.656684:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656686:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204647.656688:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204647.656690:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204647.656692:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204647.656695:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204647.656697:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204647.656703:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204647.656707:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656709:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.656712:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656714:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204647.656717:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204647.656720:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656724:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204647.656726:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204647.656731:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204647.656736:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.656736:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204647.656739:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204647.656747:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1788204647.656748:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788204647.656753:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1788204647.656755:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.656760:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204647.656770:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000100:00000040:2.0:1788204647.656770:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204647.656778:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204647.656789:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204647.656793:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.656800:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204647.656805:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204647.656810:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.656815:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1788204647.656819:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:2.0:1788204647.656830:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.656836:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204647.656839:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.656843:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204647.656850:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204647.656852:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788204647.656861:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205883520:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204647.656876:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204647.656881:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788204647.656884:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204647.656886:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.656889:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.656896:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1788204647.656897:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204647.656909:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:0.0:1788204647.656912:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.656915:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204647.656918:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204647.656923:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204647.656925:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204647.656926:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204647.656929:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204647.656932:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6102 00000400:00000200:2.0:1788204647.656944:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204647.656945:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204647.656954:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:0.0:1788204647.656961:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204647.656965:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1788204647.656968:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000040:0.0:1788204647.656970:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656975:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.656977:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.656980:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:2.0:1788204647.656980:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883520, portal 25 00000100:00000001:2.0:1788204647.656991:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788204647.656994:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204647.656996:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000020:00000040:0.0:1788204647.656998:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204647.657003:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204647.657005:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204647.657011:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204647.657013:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204647.657020:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204647.657036:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204647.657037:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204647.657044:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204647.657046:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204647.657054:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:2.0:1788204647.657054:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000001:0.0:1788204647.657063:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:2.0:1788204647.657066:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205883520, offset 0 00000020:00000040:0.0:1788204647.657069:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204647.657075:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204647.657079:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657081:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204647.657084:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204647.657090:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204647.657092:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.657096:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204647.657099:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204647.657102:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657104:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204647.657106:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204647.657113:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000400:00000200:2.0:1788204647.657118:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:0.0:1788204647.657119:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204647.657128:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204647.657131:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657133:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204647.657136:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204647.657139:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204647.657142:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657144:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000010:0.0:1788204647.657148:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000400:00000200:2.0:1788204647.657153:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788204647.657156:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657164:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00100000:0.0:1788204647.657172:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204647.657181:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58758:58758:256:4294967295] 192.168.201.151@tcp LPNI seq info [58738:58738:8:4294967295] 00000100:00000001:0.0:1788204647.657201:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204647.657202:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204647.657209:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204647.657217:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204647.657233:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204647.657247:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204647.657252:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1788204647.657261:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204647.657263:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204647.657267:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204647.657268:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:2.0:1788204647.657271:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000100:00000001:0.0:1788204647.657272:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204647.657280:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6102 00000800:00000200:2.0:1788204647.657293:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000400:00000200:0.0:1788204647.657294:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:2.0:1788204647.657310:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000400:00000200:0.0:1788204647.657311:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204647.657317:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1788204647.657327:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204647.657328:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204647.657341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1788204647.657342:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204647.657350:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000100:2.0:1788204647.657357:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204647.657360:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000010:2.0:1788204647.657405:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000020:00000001:0.0:1788204647.657411:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204647.657414:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204647.657417:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204647.657418:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:0.0:1788204647.657421:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204647.657428:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204647.657429:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1788204647.657439:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204647.657443:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204647.657450:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000040:0.0:1788204647.657454:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204647.657456:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204647.657459:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1788204647.657468:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204647.657469:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204647.657476:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204647.657485:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204647.657496:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204647.657503:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.657507:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788204647.657508:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204647.657512:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204647.657518:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.657523:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:2.0:1788204647.657524:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788204647.657527:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204647.657537:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.657544:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204647.657553:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204647.657558:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204647.657558:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204647.657563:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204647.657564:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1788204647.657567:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204647.657571:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204647.657573:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204647.657575:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.657575:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204647.657586:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000400:00000200:2.0:1788204647.657586:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204647.657591:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300779010 00000100:00000001:0.0:1788204647.657594:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204647.657597:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1420. 00000100:00100000:0.0:1788204647.657605:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204647.657606:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204647.657627:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204647.657629:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300779010 00000100:00000001:0.0:1788204647.657631:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204647.657634:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c60. 00000100:00000001:2.0:1788204647.657639:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1788204647.657640:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657644:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204647.657649:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.657658:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204647.657666:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204647/real 1788204647] req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204647.657672:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204647.657679:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204647.657682:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1788204647.657683:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6112) 00000400:00000010:2.0:1788204647.657686:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204647.657692:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204647.657694:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204647.657698:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204647.657703:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204647.657712:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657718:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204647.657720:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657723:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.657726:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204647.657730:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657736:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204647.657740:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657747:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204647.657750:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657751:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.657752:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657755:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204647.657757:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204647.657761:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204647.657766:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204647.657770:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204647.657775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204647.657781:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205883520:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204647.657785:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657788:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.657789:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205883520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657794:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204647.657797:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204647.657799:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204647.657802:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204647.657805:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204647.657806:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204647.657808:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204647.657810:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788204647.657814:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204647.657815:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204647.657817:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204647.657819:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204647.657820:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657822:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.657829:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204647.657831:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.657832:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204647.657834:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204647.657835:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204647.657838:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657842:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204647.657844:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.657845:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204647.657847:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204647.657849:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204647.657850:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883648:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204647.657853:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204647.657855:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204647.657856:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.657858:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204647.657860:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204647.657862:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.657865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204647.657867:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204647.657869:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204647.657871:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883648, portal 10 00000100:00000001:2.0:1788204647.657873:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204647.657874:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204647.657880:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657886:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204647.657888:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204647.657891:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204647.657893:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205883648, offset 0 00000400:00000200:2.0:1788204647.657897:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204647.657902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.657908:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204647.657911:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204647.657914:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204647.657916:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204647.657918:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204647.657921:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58759:58759:256:4294967295] 192.168.201.151@tcp LPNI seq info [58739:58739:8:4294967295] 00000400:00000200:2.0:1788204647.657925:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.657931:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204647.657935:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.657940:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204647.657944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204647.657947:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204647.657952:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204647.657955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204647.657959:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.657961:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204647.657964:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204647.657967:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204647.657970:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204647.657971:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204647.657972:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204647.657975:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657980:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.657982:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.657987:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.657988:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204647.657990:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204647.657993:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204647.657995:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204647.657997:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300775067 00000100:00000001:2.0:1788204647.657999:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658001:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300775067 00000100:00000001:2.0:1788204647.658002:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658003:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204647.658004:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658006:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204647.658009:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204647/real 1788204647] req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204647.658022:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204647.658024:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204647.658026:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204647.658028:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204647.658029:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204647.658031:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204647.658035:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204647.658043:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658049:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204647.658050:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658052:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658053:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204647.658056:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658062:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204647.658067:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658073:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204647.658076:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658077:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658078:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658080:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204647.658082:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204647.658084:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204647.658087:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204647.658088:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204647.658092:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204647.658097:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883648:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204647.658100:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204647.658102:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658106:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204647.658107:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.658109:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204647.658110:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204647.658111:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204647.658113:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204647.658115:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204647.658116:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204647.658117:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.658118:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204647.658121:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204647.658122:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.658124:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204647.658126:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204647.658128:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204647.658131:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883776, portal 4 00000100:00000001:2.0:1788204647.658132:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204647.658133:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204647.658137:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658141:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204647.658143:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204647.658144:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204647.658146:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205883776, offset 0 00000400:00000200:2.0:1788204647.658149:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204647.658151:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.658153:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204647.658155:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204647.658157:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204647.658159:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204647.658160:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204647.658162:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58760:58760:256:4294967295] 192.168.201.151@tcp LPNI seq info [58740:58740:8:4294967295] 00000400:00000200:2.0:1788204647.658165:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.658169:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204647.658171:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.658173:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204647.658176:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204647.658178:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204647.658180:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204647.658182:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204647.658184:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.658186:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204647.658187:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204647.658190:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204647.658192:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204647.658193:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204647.658195:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204647.658197:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658201:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.658203:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658208:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658209:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204647.658210:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204647.658213:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204647.658214:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204647.658217:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300774953 00000100:00000001:2.0:1788204647.658218:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658220:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300774953 00000100:00000001:2.0:1788204647.658221:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658222:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204647.658223:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658224:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204647.658226:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204647/real 1788204647] req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204647.658230:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204647.658231:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204647.658232:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204647.658233:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204647.658234:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204647.658235:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204647.658237:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204647.658241:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658245:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204647.658245:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658246:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658247:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204647.658249:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658252:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204647.658254:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658257:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204647.658259:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658260:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658261:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658262:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204647.658264:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204647.658266:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204647.658268:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204647.658269:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204647.658271:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204647.658274:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204647.658277:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204647.658279:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658282:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204647.658283:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.658284:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204647.658285:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204647.658287:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204647.658288:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204647.658290:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204647.658291:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204647.658292:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204647.658293:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204647.658295:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204647.658296:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204647.658298:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204647.658299:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204647.658300:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204647.658302:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205883904, portal 4 00000100:00000001:2.0:1788204647.658304:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204647.658305:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204647.658308:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658313:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204647.658315:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204647.658317:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204647.658319:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205883904, offset 0 00000400:00000200:2.0:1788204647.658323:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204647.658326:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.658329:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204647.658331:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204647.658334:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204647.658336:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204647.658338:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204647.658341:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58761:58761:256:4294967295] 192.168.201.151@tcp LPNI seq info [58741:58741:8:4294967295] 00000400:00000200:2.0:1788204647.658345:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204647.658351:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204647.658356:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.658359:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204647.658362:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204647.658393:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204647.658398:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204647.658400:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204647.658404:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204647.658406:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204647.658409:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204647.658412:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204647.658415:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204647.658416:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204647.658418:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204647.658420:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658429:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.658431:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658435:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658436:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204647.658438:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204647.658440:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204647.658443:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204647.658444:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300774960 00000100:00000001:2.0:1788204647.658446:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658447:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300774960 00000100:00000001:2.0:1788204647.658449:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300779010 00000100:00000001:2.0:1788204647.658450:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204647.658451:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658452:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204647.658454:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204647/real 1788204647] req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204647.658458:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204647.658459:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204647.658461:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204647.658463:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204647.658464:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204647.658466:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204647.658469:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204647.658536:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658544:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204647.658546:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658548:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658549:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204647.658554:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658560:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204647.658564:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658568:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204647.658570:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658571:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204647.658572:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658573:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204647.658574:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204647.658576:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204647.658577:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204647.658578:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204647.658581:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204647.658586:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205883904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204647.658588:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658589:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.658591:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205883648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658594:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204647.658595:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204647.658597:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204647.658599:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204647.658600:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204647.658601:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204647.658602:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204647.658604:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:2.0:1788204647.658606:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204647.658607:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204647.658608:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204647.658609:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204647.658610:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658612:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.658614:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205883776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658631:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204647.658632:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204647.658633:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204647.658635:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204647.658635:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204647.658636:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204647.658638:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204647.658639:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:2.0:1788204647.658640:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204647.658641:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204647.658642:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204647.658643:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204647.658643:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658644:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204647.658646:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205883904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204647.658649:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204647.658650:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204647.658651:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204647.658652:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204647.658652:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204647.658653:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204647.658654:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204647.658655:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:2.0:1788204647.658657:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204647.658658:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204647.658659:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204647.658660:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204647.658661:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658662:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.658665:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204647.658666:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.658667:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204647.658668:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204647.848491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204647.848496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204647.848499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204647.848500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204647.848502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204647.848503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204647.848505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204647.848506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204647.848519:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204647.848521:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204647.848523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204648.616464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204648.616470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204648.616471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204648.616475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204648.616477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204648.616480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204648.616482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204648.616483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204648.616485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204648.680464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204648.680470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204648.680473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204648.680478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204648.680482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204648.680486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204648.680489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204648.680491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204648.680494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204648.872460:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204648.872466:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204648.872471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204648.872473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204648.872476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204648.872478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204648.872482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204648.872483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204648.872540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204648.872545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204648.872548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204649.640469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204649.640474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204649.640475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204649.640479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204649.640481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204649.640484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204649.640486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204649.640488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204649.640490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204649.704455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204649.704462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204649.704465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204649.704470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204649.704474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204649.704477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204649.704480:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204649.704482:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204649.704486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204649.704486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204649.704491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204649.704493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204649.704497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204649.704499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204649.704502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204649.704504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204649.704505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204649.704507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204649.896489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204649.896494:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204649.896496:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204649.896498:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896500:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204649.896502:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896503:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896504:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204649.896507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204649.896509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204649.896509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204649.896526:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204649.896527:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204649.896529:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204650.216455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204650.216462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204650.216464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204650.216468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204650.216472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204650.216476:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204650.216479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204650.216482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204650.216485:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204650.664454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204650.664462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204650.664465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204650.664470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204650.664475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204650.664478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204650.664481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204650.664484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204650.664487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204650.728457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204650.728468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204650.728469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204650.728474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204650.728474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204650.728477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204650.728481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204650.728482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204650.728486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204650.728488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204650.728490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204650.728493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204650.728494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204650.728498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204650.728502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204650.728506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204650.728510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204650.728514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204650.920473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204650.920479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204650.920500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204650.920501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204650.920503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204650.920504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204650.920506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204650.920507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204650.920557:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204650.920574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204650.920576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204651.240455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204651.240462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204651.240464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204651.240469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204651.240472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204651.240475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204651.240478:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204651.240480:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204651.240483:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204651.688472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204651.688478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204651.688480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204651.688484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204651.688487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204651.688490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204651.688492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204651.688494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204651.688496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204651.752460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204651.752460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204651.752467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204651.752467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204651.752471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204651.752472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204651.752478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204651.752479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204651.752484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204651.752484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204651.752491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204651.752491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204651.752497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204651.752499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204651.752502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204651.752503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204651.752506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204651.752508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204651.944560:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204651.944563:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204651.944566:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944572:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204651.944574:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944576:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944579:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944584:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204651.944586:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944589:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204651.944591:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204651.944592:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204651.944594:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944595:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204651.944597:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944599:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944600:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944602:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204651.944604:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204651.944605:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204651.944606:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944607:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204651.944607:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944608:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204651.944610:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204651.944625:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204651.944627:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204651.944629:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204652.264454:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300779010, at 4300779520 00000100:00000001:2.0:1788204652.264463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204652.264468:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300779010, at 4300779520 00000100:00000010:1.0:1788204652.264470:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000001:2.0:1788204652.264471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204652.264477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204652.264479:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00100000:2.0:1788204652.264501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204652.264506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204652.264509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204652.264512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204652.264513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204652.264516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204652.712449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204652.712456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204652.712458:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204652.712464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204652.712468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204652.712472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204652.712475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204652.712477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204652.712481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204652.776455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204652.776461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204652.776462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204652.776466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204652.776468:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204652.776474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204652.776476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204652.776477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204652.776482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204652.776483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204652.776484:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204652.776487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204652.776488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204652.776493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204652.776496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204652.776496:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204652.776501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776502:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204652.776504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204652.776508:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204652.776509:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204652.776512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776516:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204652.776542:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6107 00000400:00000200:2.0:1788204652.776551:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204652.776556:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204652.776559:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204652.776562:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776566:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.776569:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776572:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204652.776576:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204652.776579:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204652.776582:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204652.776586:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204652.776589:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204652.776593:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204652.776597:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776600:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.776605:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776608:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204652.776612:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204652.776615:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776618:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204652.776620:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204652.776630:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204652.776633:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776637:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204652.776640:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204652.776648:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204652.776658:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204652.776663:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204652.776666:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776669:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204652.776672:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204652.776677:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204652.776685:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204652.776693:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204652.776703:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205884032/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204652.776714:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204652.776718:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204652.776724:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205884032/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204652.776748:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204652.776751:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204652.776754:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204652.776755:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204652.776756:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204652.776758:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204652.776760:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6107 00000400:00000200:2.0:1788204652.776763:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204652.776765:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204652.776768:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204652.776770:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776772:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.776774:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776776:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204652.776779:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204652.776781:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204652.776783:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204652.776786:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204652.776792:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204652.776797:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204652.776806:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776810:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.776816:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776822:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204652.776829:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204652.776837:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204652.776841:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204652.776844:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204652.776850:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204652.776854:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776858:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204652.776863:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204652.776868:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776873:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204652.776877:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204652.776881:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204652.776886:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204652.776890:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.776895:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204652.776896:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204652.776899:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204652.776908:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788204652.776915:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204652.776926:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205884160/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204652.776938:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204652.776942:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204652.776947:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205884160/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204652.776969:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204652.776974:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204652.776982:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204652.776985:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204652.776988:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204652.776994:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204652.776999:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6107 00000400:00000200:2.0:1788204652.777005:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204652.777010:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204652.777015:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204652.777021:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777026:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.777029:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777034:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204652.777040:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204652.777042:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204652.777046:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204652.777050:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204652.777052:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204652.777055:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204652.777059:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777063:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.777066:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777070:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204652.777072:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204652.777078:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777081:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204652.777084:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204652.777089:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204652.777092:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777096:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204652.777098:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204652.777102:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777106:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204652.777111:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204652.777117:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204652.777122:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204652.777123:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777125:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204652.777127:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204652.777129:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204652.777135:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204652.777141:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204652.777148:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205884288/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204652.777161:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204652.777166:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204652.777173:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205884288/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204652.777191:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204652.777191:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:2.0:1788204652.777198:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204652.777199:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00080000:2.0:1788204652.777204:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204652.777207:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204652.777211:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204652.777217:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204652.777221:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6107 00000400:00000200:2.0:1788204652.777226:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204652.777230:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204652.777234:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204652.777240:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777242:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.777247:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777250:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204652.777252:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204652.777254:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204652.777257:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204652.777261:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204652.777263:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204652.777265:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204652.777269:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777273:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204652.777278:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777282:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204652.777287:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204652.777293:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204652.777297:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204652.777302:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204652.777307:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204652.777314:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777319:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204652.777322:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204652.777327:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777332:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204652.777338:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204652.777343:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204652.777348:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204652.777350:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.777354:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204652.777355:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204652.777358:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204652.777419:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204652.777427:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204652.777439:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205884416/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204652.777460:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204652.777465:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204652.777474:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205884416/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204652.777492:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204652.777500:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6117) 00000800:00000200:3.1:1788204652.777725:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204652.777727:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204652.777933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204652.777935:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204652.778124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204652.778126:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00010000:00000001:1.0:1788204652.968471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204652.968477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204652.968481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204652.968483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204652.968485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204652.968487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204652.968490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204652.968491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204652.968565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204652.968570:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204652.968575:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204653.288500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204653.288507:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204653.288510:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204653.288515:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204653.288533:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204653.288538:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204653.288541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204653.288543:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204653.288546:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204653.736461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204653.736467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204653.736469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204653.736473:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204653.736476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204653.736479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204653.736482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204653.736484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204653.736487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204653.800495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204653.800497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204653.800502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204653.800504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204653.800506:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204653.800508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204653.800522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204653.800524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204653.800529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204653.800530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204653.800534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204653.800535:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204653.800541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204653.800544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204653.800548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204653.800549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204653.800553:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204653.800556:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204653.992529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204653.992536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204653.992542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204653.992544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204653.992547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204653.992549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204653.992552:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204653.992553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204653.992568:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204653.992570:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204653.992572:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204654.312480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204654.312489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204654.312492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204654.312497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204654.312526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204654.312529:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204654.312532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204654.312534:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204654.312536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204654.760445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204654.760451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204654.760453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204654.760457:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204654.760460:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204654.760463:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204654.760465:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204654.760467:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204654.760471:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204654.824452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204654.824456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204654.824459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204654.824459:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204654.824462:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204654.824462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204654.824468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204654.824473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204654.824476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204654.824477:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204654.824479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204654.824480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204654.824481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204654.824484:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204654.824487:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204654.824489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204654.824490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204654.824505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204655.016468:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204655.016475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204655.016482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204655.016484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204655.016488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204655.016522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204655.016526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204655.016528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204655.016598:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204655.016602:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204655.016605:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204655.336505:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204655.336513:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204655.336515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204655.336519:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204655.336522:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204655.336525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204655.336527:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204655.336529:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204655.336532:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204655.784469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204655.784500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204655.784501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204655.784506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204655.784508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204655.784511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204655.784513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204655.784515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204655.784518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204655.848506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204655.848506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204655.848515:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204655.848515:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204655.848520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204655.848520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204655.848529:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204655.848529:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204655.848534:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204655.848536:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204655.848539:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204655.848541:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204655.848542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204655.848546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204655.848548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204655.848554:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204655.848558:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204655.848562:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204656.040586:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204656.040594:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204656.040600:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204656.040602:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204656.040606:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204656.040607:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204656.040611:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204656.040613:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204656.040637:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204656.040640:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204656.040644:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204656.360455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204656.360461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204656.360463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204656.360489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204656.360492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204656.360496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204656.360498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204656.360500:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204656.360502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204656.808437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204656.808443:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204656.808444:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204656.808448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204656.808467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204656.808475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204656.808478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204656.808481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204656.808484:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204656.872443:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204656.872450:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204656.872452:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204656.872470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204656.872473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204656.872476:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204656.872479:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204656.872481:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204656.872484:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204656.872487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204656.872492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204656.872493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204656.872497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204656.872500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204656.872502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204656.872505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204656.872508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204656.872511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204657.064473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204657.064479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204657.064484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204657.064486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204657.064488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204657.064489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204657.064492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204657.064493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204657.064558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204657.064563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204657.064566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204657.384497:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300784130, at 4300784640 00000100:00000001:2.0:1788204657.384497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204657.384504:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204657.384508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.384511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204657.384514:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000001:2.0:1788204657.384515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788204657.384517:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300784129, at 4300784640 00000100:00000001:2.0:1788204657.384519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204657.384524:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204657.384528:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204657.384530:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000001:2.0:1788204657.384531:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204657.832499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204657.832517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204657.832520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204657.832526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204657.832530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204657.832534:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204657.832537:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204657.832539:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204657.832542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204657.896483:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204657.896485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204657.896485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204657.896493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204657.896493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204657.896495:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204657.896498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204657.896498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896500:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204657.896505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204657.896507:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:2.0:1788204657.896507:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204657.896511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204657.896513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204657.896517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204657.896517:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204657.896518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204657.896521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204657.896524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204657.896527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204657.896529:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6112 00000100:00000001:0.0:1788204657.896532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204657.896536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204657.896540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204657.896542:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204657.896551:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204657.896555:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204657.896559:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896563:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896566:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896569:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204657.896573:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204657.896575:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204657.896578:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204657.896581:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204657.896583:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204657.896585:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204657.896588:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896590:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896593:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896594:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204657.896597:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204657.896600:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896602:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204657.896605:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204657.896608:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204657.896610:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896612:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204657.896615:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204657.896623:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204657.896625:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204657.896629:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204657.896630:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896634:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204657.896636:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204657.896641:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204657.896650:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204657.896657:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.896667:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205884544/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204657.896679:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204657.896683:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204657.896695:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205884544/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204657.896713:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204657.896715:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204657.896718:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204657.896719:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204657.896720:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204657.896723:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204657.896724:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6112 00000400:00000200:2.0:1788204657.896727:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204657.896730:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204657.896732:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204657.896735:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896737:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896739:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896740:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204657.896743:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204657.896744:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204657.896748:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204657.896751:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204657.896755:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204657.896760:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204657.896763:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896766:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896770:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896774:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204657.896784:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204657.896789:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896792:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204657.896795:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204657.896800:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204657.896803:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896808:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204657.896812:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204657.896817:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896818:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204657.896822:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204657.896824:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204657.896825:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204657.896826:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.896827:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204657.896830:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204657.896833:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204657.896839:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204657.896846:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.896857:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205884672/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204657.896873:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204657.896878:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204657.896888:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205884672/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204657.896903:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204657.896909:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204657.896914:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204657.896917:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204657.896919:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204657.896922:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204657.896926:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6112 00000400:00000200:2.0:1788204657.896935:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204657.896940:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204657.896946:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204657.896950:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896954:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896959:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896962:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204657.896963:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204657.896966:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204657.896969:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204657.896971:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204657.896975:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204657.896978:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204657.896980:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896984:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.896988:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896991:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204657.896993:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204657.896995:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204657.896997:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204657.897001:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204657.897005:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204657.897008:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897013:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204657.897019:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204657.897025:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897031:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204657.897039:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204657.897045:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204657.897053:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204657.897056:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897062:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204657.897066:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204657.897070:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204657.897075:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204657.897080:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.897088:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205884800/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204657.897102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204657.897107:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788204657.897116:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204657.897120:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204657.897127:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205884800/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204657.897142:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204657.897146:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204657.897150:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204657.897152:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204657.897155:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204657.897162:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204657.897167:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6112 00000400:00000200:2.0:1788204657.897175:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204657.897182:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204657.897187:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204657.897193:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.897197:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.897203:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.897205:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204657.897210:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204657.897213:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204657.897216:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204657.897219:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204657.897223:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204657.897225:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204657.897231:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.897232:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204657.897234:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204657.897239:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204657.897242:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204657.897246:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204657.897250:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204657.897253:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204657.897258:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204657.897261:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897264:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204657.897267:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204657.897273:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897279:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204657.897286:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204657.897293:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204657.897300:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204657.897304:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.897310:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204657.897314:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204657.897318:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204657.897325:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204657.897330:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.897336:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205884928/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204657.897346:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204657.897349:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204657.897358:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205884928/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204657.897394:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204657.897399:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6122) 00000800:00000200:3.1:1788204657.897621:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204657.897623:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204657.897811:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204657.897812:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204657.897995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204657.897997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00010000:00000001:1.0:1788204658.088549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204658.088556:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204658.088562:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204658.088564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204658.088567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204658.088568:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204658.088572:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204658.088573:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204658.088593:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204658.088596:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204658.088598:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204658.408480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204658.408487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204658.408488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204658.408492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204658.408495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204658.408498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204658.408501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204658.408503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204658.408505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204658.856493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204658.856499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204658.856501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204658.856505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204658.856508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204658.856513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204658.856515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204658.856518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204658.856521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204658.920482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204658.920482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204658.920490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204658.920490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204658.920493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204658.920493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204658.920500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204658.920500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204658.920507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204658.920507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204658.920513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204658.920513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204658.920519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204658.920521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204658.920526:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204658.920526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204658.920530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204658.920533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204659.112490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204659.112500:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204659.112503:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204659.112507:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112510:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204659.112512:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112514:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112516:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112520:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204659.112521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204659.112525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204659.112526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204659.112598:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204659.112602:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204659.112605:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204659.432482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204659.432490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204659.432492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.432497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204659.432501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204659.432505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204659.432508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204659.432512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204659.432515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204659.880500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204659.880507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204659.880508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204659.880512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204659.880515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204659.880518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204659.880520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204659.880522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204659.880524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204659.944504:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204659.944507:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204659.944512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204659.944513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204659.944518:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204659.944518:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204659.944527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204659.944531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204659.944534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204659.944535:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204659.944538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204659.944540:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204659.944543:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204659.944545:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204659.944548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204659.944549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204659.944553:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204659.944557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204660.136525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204660.136530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204660.136535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204660.136536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204660.136538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204660.136539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204660.136542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204660.136543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204660.136556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204660.136558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204660.136561:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204660.456530:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204660.456536:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204660.456538:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204660.456542:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204660.456545:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204660.456548:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204660.456550:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204660.456552:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204660.456556:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204660.904484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204660.904490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204660.904492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204660.904496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204660.904499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204660.904510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204660.904512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204660.904514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204660.904516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204660.968492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204660.968493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204660.968499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204660.968500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204660.968502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204660.968504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204660.968508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204660.968511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204660.968513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204660.968517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204660.968519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204660.968521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204660.968523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204660.968525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204660.968530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204660.968533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204660.968534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204660.968538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204661.160471:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204661.160474:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204661.160478:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160483:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204661.160485:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160488:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160490:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204661.160496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204661.160501:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204661.160503:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204661.160504:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160506:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204661.160507:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160509:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160511:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204661.160517:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204661.160518:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204661.160520:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160521:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204661.160523:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160525:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204661.160526:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204661.160583:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204661.160587:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204661.160590:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204661.480488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204661.480495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204661.480497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204661.480503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204661.480506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204661.480511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204661.480514:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204661.480516:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204661.480519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204661.928479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204661.928486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204661.928487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204661.928491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204661.928494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204661.928497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204661.928499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204661.928501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204661.928503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204661.992479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204661.992483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204661.992486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204661.992488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204661.992489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204661.992492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204661.992497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204661.992500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204661.992502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204661.992504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204661.992506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204661.992510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204661.992511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204661.992515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204661.992519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204661.992522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204661.992526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204661.992529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204662.184536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204662.184541:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204662.184554:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204662.184555:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204662.184557:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204662.184561:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204662.184563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204662.184564:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204662.184579:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204662.184581:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204662.184583:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204662.504477:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300789249, at 4300789760 00000100:00000040:1.0:1788204662.504477:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300789249, at 4300789760 00000100:00000001:2.0:1788204662.504479:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204662.504486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204662.504489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204662.504492:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000010:0.0:1788204662.504494:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00100000:2.0:1788204662.504495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204662.504501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204662.504505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204662.504508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204662.504511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204662.504514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204662.952481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204662.952489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204662.952491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204662.952496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204662.952500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204662.952504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204662.952506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204662.952509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204662.952512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204663.016462:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204663.016467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204663.016471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204663.016473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204663.016475:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204663.016476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204663.016477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204663.016479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016481:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204663.016485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204663.016486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204663.016487:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204663.016490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204663.016493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204663.016496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204663.016497:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204663.016498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204663.016501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204663.016504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204663.016506:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204663.016507:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6117 00000100:00000001:1.0:1788204663.016510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204663.016514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204663.016518:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204663.016520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204663.016528:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204663.016532:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204663.016537:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016541:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016544:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016547:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204663.016551:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204663.016554:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204663.016558:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204663.016561:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204663.016564:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204663.016566:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204663.016569:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016571:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016573:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016575:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204663.016578:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204663.016581:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016584:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204663.016587:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204663.016590:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204663.016593:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016597:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204663.016599:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204663.016608:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204663.016611:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204663.016614:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204663.016616:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016618:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204663.016620:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204663.016623:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204663.016629:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788204663.016635:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204663.016643:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205885056/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204663.016651:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204663.016654:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204663.016657:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205885056/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204663.016677:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204663.016681:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.016685:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204663.016686:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204663.016688:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204663.016691:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204663.016694:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6117 00000400:00000200:2.0:1788204663.016699:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204663.016702:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204663.016704:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204663.016707:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016709:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016711:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016714:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204663.016717:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204663.016718:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204663.016720:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204663.016722:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204663.016724:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204663.016726:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204663.016729:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016732:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016734:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016736:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204663.016738:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204663.016741:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016744:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204663.016746:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204663.016751:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204663.016753:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016755:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204663.016760:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204663.016763:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016765:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204663.016768:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204663.016771:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204663.016773:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204663.016775:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016777:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204663.016778:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204663.016780:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204663.016784:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788204663.016787:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204663.016793:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205885184/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204663.016800:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204663.016803:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204663.016807:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205885184/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204663.016814:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204663.016817:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.016821:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204663.016822:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204663.016824:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204663.016827:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204663.016830:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6117 00000400:00000200:2.0:1788204663.016833:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204663.016836:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204663.016839:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204663.016842:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016844:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016846:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016848:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204663.016850:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204663.016851:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204663.016853:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204663.016855:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204663.016857:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204663.016859:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204663.016862:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016864:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016866:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016868:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204663.016870:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204663.016873:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016875:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204663.016877:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204663.016880:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204663.016882:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016884:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204663.016886:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204663.016889:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016891:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204663.016893:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204663.016896:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204663.016898:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204663.016900:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.016901:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204663.016903:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204663.016904:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204663.016909:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204663.016912:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204663.016917:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205885312/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204663.016925:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204663.016927:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204663.016928:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:2.0:1788204663.016930:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205885312/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204663.016932:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788204663.016939:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204663.016941:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.016944:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204663.016946:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204663.016947:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204663.016950:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204663.016952:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6117 00000400:00000200:2.0:1788204663.016956:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204663.016959:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204663.016961:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204663.016964:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016966:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016968:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016970:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204663.016973:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204663.016974:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204663.016975:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204663.016978:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204663.016980:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204663.016982:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204663.016986:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016988:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204663.016990:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204663.016992:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204663.016994:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204663.016997:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204663.017000:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204663.017002:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204663.017004:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204663.017006:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.017008:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204663.017010:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204663.017013:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.017015:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204663.017019:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204663.017021:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204663.017023:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204663.017025:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.017026:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204663.017028:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204663.017029:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204663.017033:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204663.017036:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204663.017041:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205885440/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204663.017048:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204663.017050:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204663.017054:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205885440/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204663.017063:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204663.017067:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6127) 00000800:00000200:3.1:1788204663.017275:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204663.017277:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204663.017524:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204663.017526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204663.017720:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204663.017722:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00010000:00000001:1.0:1788204663.208476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204663.208482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204663.208486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204663.208488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204663.208490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204663.208491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204663.208494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204663.208496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204663.208610:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204663.208614:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204663.208617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204663.528456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204663.528464:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204663.528466:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204663.528472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204663.528475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204663.528479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204663.528482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204663.528484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204663.528487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204663.976470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204663.976478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204663.976480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204663.976486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204663.976490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204663.976494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204663.976496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204663.976498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204663.976502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204664.040472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204664.040472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204664.040479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204664.040479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204664.040483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204664.040483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204664.040490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204664.040490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204664.040496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204664.040497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204664.040501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204664.040502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204664.040507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204664.040509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204664.040511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204664.040513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204664.040516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204664.040519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204664.232501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204664.232506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204664.232510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204664.232511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204664.232513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204664.232514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204664.232516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204664.232517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204664.232532:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204664.232534:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204664.232536:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204665.000474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204665.000482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204665.000484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204665.000490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204665.000495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204665.000499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204665.000502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204665.000504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204665.000507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204665.064492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204665.064501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204665.064507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204665.064513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204665.064517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204665.064521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204665.064524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204665.064527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204665.064528:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204665.064531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204665.064533:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204665.064537:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204665.064541:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204665.064545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204665.064548:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204665.064551:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204665.064553:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204665.064556:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204665.256476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204665.256482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204665.256487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204665.256490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204665.256492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204665.256493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204665.256496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204665.256497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204665.256564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204665.256568:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204665.256571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204666.024441:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204666.024448:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204666.024450:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204666.024456:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204666.024460:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204666.024464:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204666.024467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204666.024470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204666.024473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204666.088471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204666.088472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204666.088480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204666.088480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204666.088484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204666.088484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204666.088491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204666.088491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204666.088495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204666.088496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204666.088499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204666.088500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204666.088504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204666.088508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204666.088510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204666.088511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204666.088514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204666.088517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204666.280527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204666.280532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204666.280536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204666.280537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204666.280539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204666.280540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204666.280542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204666.280543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204666.280558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204666.280560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204666.280562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204667.048498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204667.048506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204667.048509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204667.048516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204667.048521:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204667.048526:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204667.048529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204667.048532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204667.048535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204667.112445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204667.112450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204667.112451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204667.112456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204667.112458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204667.112464:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204667.112464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204667.112468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204667.112470:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204667.112475:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204667.112476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204667.112481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204667.112481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204667.112487:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204667.112492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204667.112493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204667.112496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204667.112500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204667.304477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204667.304483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204667.304488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204667.304490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204667.304494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204667.304496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204667.304500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204667.304501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204667.304574:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204667.304578:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204667.304581:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204667.624461:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300794369, at 4300794880 00000100:00000040:1.0:1788204667.624462:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300794369, at 4300794880 00000100:00000010:0.0:1788204667.624475:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000010:1.0:1788204667.624477:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00000001:1.0:1788204668.072484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204668.072491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204668.072492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204668.072496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204668.072499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204668.072502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204668.072504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204668.072505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204668.072507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204668.136463:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204668.136467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204668.136472:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204668.136473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204668.136476:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204668.136478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204668.136479:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204668.136485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204668.136486:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204668.136491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204668.136492:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6122 00000100:00000001:1.0:1788204668.136497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204668.136502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788204668.136503:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204668.136506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204668.136509:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204668.136512:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204668.136512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.136519:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136524:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.136527:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136529:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204668.136533:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204668.136535:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204668.136538:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204668.136541:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204668.136544:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204668.136547:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204668.136550:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136552:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.136554:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136556:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204668.136559:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204668.136562:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136565:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204668.136567:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204668.136570:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204668.136573:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.136575:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204668.136577:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204668.136584:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204668.136586:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204668.136589:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204668.136590:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.136592:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204668.136594:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204668.136597:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204668.136603:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204668.136608:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204668.136615:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205885568/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204668.136621:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204668.136624:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.136629:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205885568/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204668.136650:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204668.136653:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204668.136657:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204668.136658:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204668.136660:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204668.136662:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204668.136665:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6122 00000400:00000200:0.0:1788204668.136669:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204668.136678:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204668.136680:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204668.136682:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136685:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.136689:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136691:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204668.136694:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204668.136694:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204668.136695:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204668.136697:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204668.136698:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204668.136699:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204668.136702:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136705:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.136706:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136708:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204668.136710:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204668.136714:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136716:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204668.136718:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204668.136724:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204668.136728:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.136733:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204668.136739:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204668.136744:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.136748:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204668.136756:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204668.136762:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204668.136768:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204668.136772:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.136779:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204668.136783:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204668.136787:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204668.136795:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204668.136800:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204668.136809:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205885696/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204668.136824:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204668.136829:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.136838:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205885696/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204668.136858:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204668.136862:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204668.136868:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204668.136871:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204668.136873:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204668.136881:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204668.136888:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6122 00000400:00000200:0.0:1788204668.136898:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204668.136908:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204668.136914:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204668.136921:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136927:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.136933:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136938:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204668.136944:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204668.136948:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204668.136953:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204668.136955:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204668.136959:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204668.136964:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204668.136972:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.136977:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204668.136978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204668.136981:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204668.136984:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1788204668.136986:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204668.136991:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204668.136998:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137002:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204668.137005:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204668.137009:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204668.137013:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137019:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204668.137023:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204668.137027:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137031:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204668.137035:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204668.137040:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204668.137046:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204668.137049:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137053:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204668.137056:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204668.137059:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204668.137065:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204668.137071:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204668.137081:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205885824/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204668.137094:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204668.137097:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.137105:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205885824/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204668.137118:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204668.137122:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204668.137126:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204668.137130:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204668.137131:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204668.137135:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204668.137140:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6122 00000400:00000200:0.0:1788204668.137149:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204668.137156:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204668.137160:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204668.137167:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137171:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.137176:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137182:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204668.137186:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204668.137189:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204668.137193:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204668.137197:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204668.137202:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204668.137207:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204668.137215:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137219:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204668.137224:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137229:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204668.137232:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204668.137237:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204668.137243:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204668.137258:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204668.137275:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204668.137280:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137285:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204668.137288:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204668.137292:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137296:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204668.137301:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204668.137305:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204668.137310:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204668.137313:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204668.137316:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204668.137319:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204668.137323:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204668.137331:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204668.137335:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204668.137344:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205885952/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204668.137361:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204668.137390:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.137399:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205885952/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204668.137426:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204668.137434:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6133) 00000800:00000200:3.1:1788204668.137537:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204668.137539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204668.137698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204668.137699:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204668.137847:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204668.137848:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00010000:00000001:1.0:1788204668.328491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204668.328498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204668.328504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204668.328505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204668.328508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204668.328510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204668.328512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204668.328514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204668.328530:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204668.328533:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204668.328535:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204669.096456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204669.096462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204669.096464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204669.096468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204669.096471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204669.096474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204669.096476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204669.096478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204669.096479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204669.160425:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204669.160429:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204669.160431:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204669.160436:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204669.160438:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204669.160440:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204669.160441:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204669.160442:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204669.160444:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204669.352476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204669.352484:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204669.352486:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204669.352489:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352491:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204669.352493:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352494:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352496:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204669.352498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204669.352500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204669.352501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204669.352556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204669.352559:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204669.352561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204670.120472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204670.120478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204670.120479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204670.120483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204670.120486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204670.120489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204670.120491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204670.120492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204670.120494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204670.184489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204670.184496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204670.184497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204670.184502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204670.184505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204670.184506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204670.184511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204670.184511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204670.184515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204670.184515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204670.184519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204670.184521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204670.184522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204670.184526:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204670.184530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204670.184532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204670.184534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204670.184537:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204670.376500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204670.376506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204670.376512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204670.376513:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204670.376516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204670.376518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204670.376521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204670.376522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204670.376544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204670.376547:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204670.376550:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204671.144467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204671.144472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204671.144474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204671.144477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204671.144480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204671.144482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204671.144484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204671.144486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204671.144487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204671.208459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204671.208465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204671.208467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204671.208469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204671.208473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204671.208475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204671.208479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204671.208479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204671.208484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204671.208485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204671.208488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204671.208490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204671.208493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204671.208494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204671.208499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204671.208500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204671.208504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204671.208508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400473:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204671.400479:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204671.400481:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204671.400484:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400489:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204671.400491:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400492:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400494:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204671.400497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204671.400501:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204671.400501:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204671.400502:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400503:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204671.400504:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400505:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400506:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204671.400510:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204671.400511:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204671.400512:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400513:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204671.400513:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400514:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204671.400515:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204671.400564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204671.400567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204671.400569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204672.168445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204672.168458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204672.168460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204672.168465:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204672.168468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204672.168472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204672.168475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204672.168477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204672.168480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204672.232480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204672.232489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204672.232492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204672.232493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204672.232502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204672.232503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204672.232506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204672.232510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204672.232513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204672.232516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204672.232520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204672.232523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204672.232527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204672.232528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204672.232533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204672.232537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204672.232540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204672.232543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204672.424488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204672.424493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204672.424496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204672.424497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204672.424499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204672.424500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204672.424503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204672.424503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204672.424515:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204672.424516:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204672.424518:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204672.744480:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300799489, at 4300800000 00000100:00000010:1.0:1788204672.744494:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00000040:0.0:1788204672.744503:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300799489, at 4300800000 00000100:00000010:0.0:1788204672.744521:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:1.0:1788204673.192458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204673.192463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204673.192464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.192468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204673.192470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204673.192473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204673.192475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204673.192476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204673.192478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204673.256482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204673.256484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.256489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204673.256492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204673.256495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204673.256496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204673.256499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204673.256501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204673.256536:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204673.256543:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204673.256546:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204673.256551:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204673.256557:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204673.256566:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6128 00000400:00000200:1.0:1788204673.256579:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204673.256586:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204673.256590:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204673.256594:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256596:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.256598:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256601:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204673.256605:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204673.256607:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204673.256609:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204673.256612:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204673.256614:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204673.256615:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204673.256617:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256619:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.256620:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256621:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204673.256623:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204673.256625:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256627:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204673.256629:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204673.256632:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204673.256633:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256635:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204673.256637:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204673.256644:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204673.256647:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204673.256650:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204673.256652:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256655:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204673.256657:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204673.256661:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204673.256669:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be800. 00000100:00000001:1.0:1788204673.256676:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.256685:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205886080/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204673.256694:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204673.256699:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204673.256703:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205886080/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204673.256725:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204673.256727:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204673.256731:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204673.256732:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204673.256734:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204673.256736:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204673.256739:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6128 00000400:00000200:1.0:1788204673.256743:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204673.256746:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204673.256749:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204673.256752:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256754:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.256757:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256762:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204673.256766:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204673.256767:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204673.256769:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204673.256771:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204673.256773:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204673.256777:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204673.256781:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256784:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.256788:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256792:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204673.256796:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204673.256800:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256805:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204673.256809:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204673.256813:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204673.256817:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256820:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204673.256824:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204673.256829:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256832:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204673.256836:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788204673.256839:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204673.256843:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204673.256845:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.256846:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204673.256847:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204673.256849:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204673.256853:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788204673.256859:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.256868:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205886208/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204673.256878:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204673.256882:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204673.256887:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205886208/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204673.256902:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204673.256908:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204673.256916:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204673.256919:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204673.256924:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204673.256929:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204673.256934:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6128 00000400:00000200:1.0:1788204673.256943:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204673.256950:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204673.256953:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204673.256960:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256965:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.256971:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.256978:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204673.256982:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204673.256985:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204673.256988:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204673.256990:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204673.256993:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204673.256997:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204673.257001:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257002:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.257006:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257009:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204673.257012:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204673.257015:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257019:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204673.257022:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204673.257024:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204673.257027:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257030:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204673.257033:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204673.257035:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257036:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204673.257041:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788204673.257043:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204673.257046:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204673.257049:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257051:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204673.257053:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204673.257056:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204673.257059:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000010:1.0:1788204673.257061:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000800:00000200:3.1:1788204673.257062:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1788204673.257067:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.257073:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205886336/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204673.257089:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204673.257092:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204673.257098:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205886336/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204673.257107:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204673.257111:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204673.257117:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204673.257120:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204673.257122:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204673.257126:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204673.257129:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6128 00000400:00000200:1.0:1788204673.257137:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204673.257144:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204673.257147:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204673.257154:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257160:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.257164:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257168:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204673.257186:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204673.257188:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204673.257189:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204673.257192:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204673.257194:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204673.257196:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204673.257199:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257202:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204673.257206:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257208:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204673.257211:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204673.257216:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204673.257222:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204673.257227:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204673.257232:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204673.257237:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257242:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204673.257245:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204673.257248:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257251:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204673.257256:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204673.257259:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204673.257265:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204673.257267:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204673.257269:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204673.257271:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204673.257274:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204673.257280:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788204673.257286:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204673.257294:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205886464/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204673.257303:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204673.257307:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204673.257313:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205886464/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204673.257330:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204673.257335:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6138) 00000800:00000200:3.1:1788204673.257524:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204673.257526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204673.257779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204673.257782:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204673.257942:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204673.257943:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00010000:00000001:1.0:1788204673.448462:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204673.448468:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204673.448472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204673.448473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204673.448475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204673.448476:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204673.448478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204673.448479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204673.448543:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204673.448546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204673.448548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204674.216460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204674.216467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204674.216469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204674.216474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204674.216477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204674.216481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204674.216492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204674.216495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204674.216497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204674.280459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204674.280466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204674.280468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204674.280473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204674.280476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204674.280479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204674.280487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204674.280489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204674.280490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204674.472492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204674.472499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204674.472504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204674.472506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204674.472509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204674.472510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204674.472513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204674.472514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204674.472531:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204674.472534:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204674.472536:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204675.240460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204675.240466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204675.240468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204675.240472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204675.240475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204675.240478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204675.240480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204675.240481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204675.240483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204675.304483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204675.304488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204675.304491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204675.304494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204675.304499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204675.304505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204675.304527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204675.304531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204675.304535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204675.496466:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204675.496473:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204675.496480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204675.496481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204675.496484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204675.496486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204675.496489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204675.496490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204675.496541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204675.496544:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204675.496547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204676.264428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204676.264434:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204676.264435:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204676.264439:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204676.264442:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204676.264444:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204676.264446:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204676.264448:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204676.264450:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204676.328448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204676.328456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204676.328460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204676.328468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204676.328475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204676.328482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204676.328489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204676.328493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204676.328497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204676.520487:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204676.520491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204676.520495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204676.520496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204676.520498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204676.520499:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204676.520501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204676.520501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204676.520514:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204676.520515:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204676.520518:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204677.288500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204677.288506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204677.288507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204677.288511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204677.288514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204677.288517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204677.288519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204677.288521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204677.288523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204677.352476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204677.352483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204677.352486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204677.352495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204677.352502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204677.352509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204677.352523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204677.352527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204677.352530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204677.544491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204677.544497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204677.544501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204677.544504:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204677.544506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204677.544507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204677.544509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204677.544510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204677.544568:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204677.544571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204677.544573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204677.864467:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300804609, at 4300805120 00000100:00000040:1.0:1788204677.864467:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300804609, at 4300805120 00000100:00000010:0.0:1788204677.864486:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000010:1.0:1788204677.864489:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000001:1.0:1788204678.312515:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204678.312525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204678.312527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204678.312534:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204678.312538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204678.312542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204678.312545:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204678.312548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204678.312551:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204678.376527:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204678.376528:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204678.376536:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204678.376540:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204678.376541:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376545:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204678.376548:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204678.376552:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204678.376554:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204678.376558:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204678.376561:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204678.376565:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204678.376570:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6133 00000100:00000001:1.0:1788204678.376570:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204678.376575:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204678.376584:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204678.376592:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204678.376596:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204678.376600:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376604:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.376608:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376610:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204678.376615:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204678.376618:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204678.376621:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204678.376624:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204678.376626:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204678.376628:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204678.376630:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376632:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.376633:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376634:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204678.376637:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204678.376640:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376641:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204678.376644:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204678.376648:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204678.376650:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376653:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204678.376656:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204678.376666:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204678.376677:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204678.376682:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204678.376686:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376689:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204678.376695:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204678.376704:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204678.376718:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204678.376728:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204678.376741:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205886592/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204678.376753:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204678.376759:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204678.376764:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205886592/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204678.376791:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204678.376794:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204678.376802:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204678.376803:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204678.376806:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204678.376812:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204678.376818:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6133 00000400:00000200:0.0:1788204678.376825:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204678.376833:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204678.376838:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204678.376845:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376852:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.376856:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376862:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204678.376869:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204678.376872:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204678.376876:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204678.376880:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204678.376885:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204678.376890:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204678.376898:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376901:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.376904:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376907:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204678.376911:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204678.376912:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204678.376917:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204678.376922:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204678.376927:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204678.376930:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376932:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204678.376936:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204678.376940:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376942:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204678.376945:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204678.376949:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204678.376953:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204678.376955:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.376959:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204678.376961:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204678.376964:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204678.376971:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204678.376979:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204678.376989:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205886720/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204678.377005:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204678.377009:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204678.377016:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205886720/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204678.377023:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204678.377027:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204678.377030:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204678.377031:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204678.377034:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204678.377041:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204678.377044:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6133 00000400:00000200:0.0:1788204678.377050:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204678.377056:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204678.377062:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204678.377068:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377074:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.377080:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377083:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204678.377098:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204678.377099:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204678.377101:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204678.377103:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204678.377106:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204678.377110:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204678.377118:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377123:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.377128:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377131:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204678.377136:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204678.377144:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377149:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204678.377154:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204678.377159:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204678.377161:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377163:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204678.377168:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204678.377172:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377176:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204678.377184:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204678.377190:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.1:1788204678.377193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204678.377195:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.1:1788204678.377198:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 02000000:00000001:0.0:1788204678.377199:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377204:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204678.377209:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204678.377213:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204678.377218:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204678.377224:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204678.377235:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205886848/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204678.377255:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204678.377260:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204678.377270:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205886848/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204678.377287:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204678.377293:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204678.377301:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204678.377304:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204678.377306:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204678.377312:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204678.377318:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6133 00000400:00000200:0.0:1788204678.377329:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204678.377334:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204678.377338:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204678.377342:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377345:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.377350:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377355:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204678.377360:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204678.377389:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204678.377393:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204678.377396:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204678.377401:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204678.377406:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204678.377413:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377418:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204678.377423:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377427:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204678.377431:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204678.377437:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204678.377442:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204678.377445:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204678.377450:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204678.377454:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377460:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204678.377464:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204678.377469:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377473:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204678.377478:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204678.377484:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204678.377489:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204678.377492:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204678.377496:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204678.377499:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204678.377503:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204678.377511:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204678.377518:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204678.377527:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205886976/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204678.377563:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204678.377568:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204678.377576:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205886976/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204678.377596:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204678.377604:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6143) 00000800:00000200:3.1:1788204678.377798:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204678.377800:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204678.378058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204678.378061:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204678.378348:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204678.378351:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00010000:00000001:1.0:1788204678.568490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204678.568497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204678.568501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204678.568502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204678.568505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204678.568505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204678.568508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204678.568509:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204678.568524:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204678.568526:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204678.568528:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204679.336454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204679.336461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204679.336463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204679.336468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204679.336471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204679.336475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204679.336477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204679.336479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204679.336481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204679.400484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204679.400492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204679.400494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204679.400502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204679.400506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204679.400511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204679.400523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204679.400526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204679.400528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204679.592518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204679.592527:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204679.592529:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204679.592533:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592535:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204679.592537:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592538:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592540:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204679.592543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204679.592546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204679.592547:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204679.592602:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204679.592608:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204679.592612:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204680.360524:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204680.360540:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204680.360543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204680.360549:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204680.360554:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204680.360558:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204680.360561:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204680.360564:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204680.360568:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204680.424510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204680.424510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204680.424518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204680.424519:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204680.424521:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204680.424522:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204680.424531:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204680.424531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204680.424536:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204680.424537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204680.424543:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204680.424543:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204680.424547:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204680.424552:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204680.424556:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204680.424556:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204680.424560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204680.424565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204680.616489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204680.616496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204680.616501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204680.616501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204680.616504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204680.616505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204680.616507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204680.616508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204680.616523:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204680.616525:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204680.616527:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204681.384479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204681.384488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204681.384491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204681.384497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204681.384501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204681.384506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204681.384509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204681.384512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204681.384516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204681.448507:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204681.448514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204681.448516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204681.448522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204681.448526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204681.448531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204681.448535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204681.448537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204681.448541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204681.448645:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204681.448647:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204681.448648:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204681.448651:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204681.448654:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204681.448656:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204681.448658:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204681.448660:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204681.448662:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204681.640515:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204681.640519:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204681.640523:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640530:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204681.640533:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640535:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640538:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204681.640544:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640548:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204681.640550:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204681.640552:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204681.640553:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640555:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204681.640556:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640558:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640560:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640563:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204681.640565:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204681.640566:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204681.640568:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640570:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204681.640571:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640572:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204681.640574:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204681.640642:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204681.640645:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204681.640648:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204682.408510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204682.408519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204682.408520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204682.408526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204682.408529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204682.408533:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204682.408535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204682.408537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204682.408539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204682.472479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204682.472479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204682.472487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204682.472487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204682.472489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204682.472491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204682.472496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204682.472497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204682.472501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204682.472502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204682.472506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204682.472506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204682.472511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204682.472515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204682.472518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204682.472518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204682.472521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204682.472525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204682.664511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204682.664521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204682.664528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204682.664529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204682.664533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204682.664534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204682.664538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204682.664539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204682.664560:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204682.664564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204682.664567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204682.984470:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300809729, at 4300810240 00000100:00000040:1.0:1788204682.984470:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300809729, at 4300810240 00000100:00000010:1.0:1788204682.984483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000010:0.0:1788204682.984486:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000001:1.0:1788204683.432472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204683.432479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204683.432481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204683.432486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204683.432490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204683.432493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204683.432495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204683.432498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204683.432504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204683.496465:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204683.496473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204683.496476:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204683.496478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204683.496480:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204683.496483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204683.496485:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204683.496489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204683.496491:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204683.496494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204683.496500:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6138 00000100:00000001:1.0:1788204683.496501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204683.496506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788204683.496511:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204683.496511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204683.496518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204683.496527:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204683.496531:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204683.496535:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496539:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496542:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496543:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204683.496546:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204683.496547:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204683.496549:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204683.496551:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204683.496553:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204683.496555:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204683.496557:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496558:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496559:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496561:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204683.496563:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204683.496565:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496567:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204683.496569:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204683.496571:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204683.496573:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.496574:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204683.496576:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204683.496585:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204683.496588:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204683.496591:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204683.496593:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.496595:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204683.496598:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204683.496602:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204683.496610:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204683.496617:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204683.496626:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205887104/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204683.496636:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204683.496641:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204683.496645:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205887104/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204683.496666:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204683.496669:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204683.496673:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204683.496674:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204683.496676:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204683.496678:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204683.496681:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6138 00000400:00000200:0.0:1788204683.496686:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204683.496689:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204683.496691:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204683.496696:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496697:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496700:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496703:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204683.496707:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204683.496709:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204683.496713:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204683.496715:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204683.496719:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204683.496724:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204683.496731:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496736:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496740:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496745:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204683.496749:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204683.496754:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496760:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204683.496765:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204683.496773:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204683.496777:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.496781:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204683.496786:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204683.496792:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.496795:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204683.496799:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204683.496803:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204683.496806:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204683.496808:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.496809:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204683.496810:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204683.496813:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204683.496825:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204683.496832:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204683.496841:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205887232/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204683.496864:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204683.496869:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204683.496878:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205887232/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204683.496890:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204683.496894:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204683.496897:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204683.496899:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204683.496901:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204683.496904:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204683.496908:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6138 00000400:00000200:0.0:1788204683.496920:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204683.496928:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204683.496932:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204683.496940:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496945:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496948:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496950:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204683.496956:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204683.496959:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204683.496961:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204683.496963:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204683.496966:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204683.496971:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204683.496977:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496983:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.496984:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.496986:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204683.496991:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204683.496993:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497009:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204683.497012:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204683.497015:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204683.497018:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.497020:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:3.1:1788204683.497020:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00100000:0.0:1788204683.497023:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204683.497026:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204683.497026:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1788204683.497029:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204683.497033:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204683.497037:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204683.497039:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204683.497042:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.497044:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204683.497046:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204683.497049:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204683.497052:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204683.497055:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204683.497063:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205887360/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204683.497073:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204683.497076:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204683.497081:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205887360/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204683.497091:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204683.497095:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204683.497101:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204683.497104:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204683.497106:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204683.497109:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204683.497112:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6138 00000400:00000200:0.0:1788204683.497118:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204683.497122:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204683.497125:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204683.497131:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497136:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.497141:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497146:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204683.497151:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204683.497154:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204683.497158:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204683.497162:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204683.497167:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204683.497172:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204683.497177:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497181:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204683.497187:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497191:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204683.497196:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204683.497211:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204683.497217:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204683.497220:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204683.497226:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204683.497230:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.497231:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204683.497233:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204683.497235:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.497237:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204683.497245:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204683.497249:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204683.497254:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204683.497257:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204683.497259:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204683.497262:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204683.497265:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204683.497271:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204683.497277:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204683.497287:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205887488/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204683.497297:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204683.497301:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204683.497310:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205887488/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204683.497329:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204683.497336:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6148) 00000800:00000200:3.1:1788204683.497601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204683.497603:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204683.497821:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204683.497823:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204683.498067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204683.498069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204683.688477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204683.688483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204683.688488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204683.688489:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204683.688493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204683.688494:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204683.688498:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204683.688500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204683.688563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204683.688566:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204683.688570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204684.456457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204684.456462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204684.456464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204684.456468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204684.456470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204684.456473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204684.456475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204684.456476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204684.456478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204684.520447:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204684.520451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204684.520454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204684.520459:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204684.520463:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204684.520469:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204684.520479:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204684.520481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204684.520483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204684.712509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204684.712516:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204684.712522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204684.712523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204684.712526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204684.712527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204684.712530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204684.712531:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204684.712548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204684.712551:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204684.712553:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204685.032475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204685.032483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204685.032485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.032490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204685.032494:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204685.032497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204685.032499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204685.032501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204685.032503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204685.480444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204685.480451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204685.480454:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204685.480459:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204685.480462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204685.480466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204685.480468:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204685.480470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204685.480471:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204685.544477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204685.544485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204685.544487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204685.544492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204685.544498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204685.544504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204685.544517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204685.544520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204685.544523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204685.736469:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204685.736474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204685.736480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204685.736481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204685.736484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204685.736486:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204685.736488:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204685.736490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204685.736558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204685.736563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204685.736566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204686.056447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204686.056453:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204686.056455:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204686.056459:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204686.056462:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204686.056465:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204686.056468:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204686.056469:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204686.056471:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204686.504455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204686.504460:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204686.504461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204686.504465:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204686.504468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204686.504470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204686.504472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204686.504473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204686.504475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204686.568431:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204686.568438:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204686.568441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204686.568446:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204686.568449:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204686.568453:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204686.568455:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204686.568458:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204686.568462:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204686.760515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204686.760521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204686.760524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204686.760525:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204686.760527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204686.760528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204686.760531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204686.760532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204686.760548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204686.760550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204686.760553:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204687.080421:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204687.080427:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204687.080428:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204687.080431:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204687.080434:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204687.080436:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204687.080438:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204687.080440:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204687.080442:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204687.528474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204687.528479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204687.528481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204687.528485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204687.528487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204687.528490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204687.528491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204687.528493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204687.528495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204687.592463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204687.592470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204687.592472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204687.592478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204687.592483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204687.592487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204687.592489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204687.592493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204687.592495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204687.784461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204687.784467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204687.784472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204687.784474:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204687.784477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204687.784478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204687.784482:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204687.784483:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204687.784540:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204687.784544:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204687.784547:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204688.104458:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300814849, at 4300815360 00000100:00000040:1.0:1788204688.104458:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300814849, at 4300815360 00000100:00000001:2.0:1788204688.104463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204688.104469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204688.104471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204688.104472:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000010:0.0:1788204688.104474:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00100000:2.0:1788204688.104477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204688.104482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204688.104485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204688.104489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204688.104493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204688.104496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204688.552471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204688.552477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204688.552479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204688.552484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204688.552488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204688.552491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204688.552494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204688.552496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204688.552499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204688.616439:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204688.616445:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204688.616449:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204688.616453:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204688.616457:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204688.616464:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6143 00000400:00000200:0.0:1788204688.616472:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204688.616478:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204688.616481:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204688.616484:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616487:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616488:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616490:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204688.616492:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204688.616494:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204688.616496:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204688.616498:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204688.616501:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204688.616503:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204688.616506:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616508:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616510:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616512:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204688.616514:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204688.616517:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616520:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204688.616523:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204688.616526:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204688.616529:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616531:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204688.616533:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204688.616543:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204688.616547:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204688.616550:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204688.616552:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616554:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204688.616555:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204688.616560:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204688.616569:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204688.616575:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204688.616584:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205887616/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204688.616593:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204688.616597:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204688.616601:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205887616/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204688.616620:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204688.616624:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204688.616627:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204688.616628:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204688.616630:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204688.616632:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204688.616635:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6143 00000400:00000200:0.0:1788204688.616652:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204688.616656:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204688.616660:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204688.616664:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616666:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616669:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616672:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204688.616675:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204688.616677:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204688.616679:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204688.616682:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204688.616685:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204688.616692:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204688.616697:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616700:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616703:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616706:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204688.616709:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204688.616716:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616720:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204688.616723:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204688.616728:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204688.616733:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616737:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204688.616742:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204688.616748:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616751:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204688.616758:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204688.616763:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204688.616768:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204688.616771:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616774:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204688.616777:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204688.616779:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204688.616785:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204688.616791:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204688.616799:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205887744/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204688.616812:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204688.616815:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204688.616821:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205887744/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204688.616831:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204688.616835:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204688.616839:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204688.616841:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204688.616843:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204688.616848:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204688.616852:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6143 00000400:00000200:0.0:1788204688.616855:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204688.616861:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204688.616863:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204688.616867:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616872:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616876:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616886:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204688.616891:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204688.616895:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204688.616900:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204688.616904:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204688.616921:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204688.616923:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204688.616927:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616929:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.616933:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616938:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204688.616942:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204688.616949:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204688.616954:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204688.616959:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204688.616963:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1788204688.616964:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204688.616969:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204688.616969:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1788204688.616974:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204688.616978:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204688.616984:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.616987:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204688.616992:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204688.616996:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204688.617000:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204688.617003:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.617005:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204688.617007:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204688.617009:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204688.617019:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204688.617024:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204688.617031:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205887872/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204688.617043:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204688.617045:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204688.617052:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205887872/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204688.617062:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204688.617065:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204688.617070:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204688.617071:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204688.617072:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204688.617074:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204688.617077:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6143 00000400:00000200:0.0:1788204688.617082:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204688.617087:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204688.617090:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204688.617094:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.617096:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.617100:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.617103:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204688.617106:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204688.617108:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204688.617110:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204688.617113:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204688.617116:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204688.617119:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204688.617125:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.617127:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204688.617132:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204688.617136:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204688.617143:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204688.617149:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204688.617155:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204688.617160:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204688.617165:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204688.617171:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.617175:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204688.617178:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204688.617183:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.617186:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204688.617192:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204688.617195:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204688.617198:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204688.617200:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204688.617203:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204688.617205:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204688.617207:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204688.617211:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204688.617216:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204688.617225:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205888000/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204688.617238:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204688.617242:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204688.617247:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205888000/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204688.617258:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204688.617262:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6153) 00000100:00000001:1.0:1788204688.617330:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204688.617334:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204688.617337:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204688.617342:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204688.617346:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204688.617349:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204688.617351:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204688.617354:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204688.617357:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204688.617436:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204688.617437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204688.617601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204688.617602:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204688.617779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204688.617781:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204688.808490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204688.808496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204688.808502:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204688.808503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204688.808505:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204688.808506:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204688.808509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204688.808510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204688.808526:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204688.808529:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204688.808532:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204689.128461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204689.128466:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204689.128468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204689.128473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204689.128476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204689.128480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204689.128482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204689.128485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204689.128487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204689.576447:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204689.576452:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204689.576453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204689.576457:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204689.576467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204689.576471:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204689.576473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204689.576476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204689.576479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204689.640460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204689.640465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204689.640469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204689.640474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204689.640480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204689.640484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204689.640488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204689.640491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204689.640494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832447:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204689.832452:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832456:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204689.832458:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204689.832460:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204689.832462:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832464:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204689.832466:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832467:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832468:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204689.832471:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204689.832474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204689.832475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204689.832526:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204689.832530:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204689.832533:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204690.152469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204690.152476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204690.152478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204690.152483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204690.152486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204690.152490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204690.152492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204690.152494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204690.152496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204690.600481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204690.600488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204690.600490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204690.600495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204690.600499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204690.600503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204690.600506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204690.600509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204690.600511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204690.664465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204690.664472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204690.664475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204690.664481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204690.664485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204690.664488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204690.664491:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204690.664494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204690.664497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204690.664497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204690.664501:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204690.664503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204690.664505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204690.664507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204690.664509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204690.664512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204690.664513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204690.664516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204690.856508:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204690.856514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204690.856519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204690.856520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204690.856523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204690.856524:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204690.856527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204690.856528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204690.856545:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204690.856548:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204690.856550:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204691.176489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204691.176496:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204691.176498:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204691.176503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204691.176506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204691.176509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204691.176512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204691.176515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204691.176517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204691.624463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204691.624469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204691.624472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204691.624477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204691.624479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204691.624482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204691.624484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204691.624485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204691.624488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204691.688453:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204691.688459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204691.688460:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204691.688464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204691.688465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204691.688468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204691.688471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204691.688472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204691.688475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204691.688477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204691.688479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204691.688481:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204691.688482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204691.688485:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204691.688488:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204691.688489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204691.688492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204691.688495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880458:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204691.880465:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204691.880467:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204691.880470:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880474:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204691.880478:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880479:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880481:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204691.880485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204691.880487:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204691.880488:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204691.880489:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880490:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204691.880491:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880492:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880493:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204691.880496:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204691.880496:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204691.880497:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880498:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204691.880499:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880500:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204691.880501:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204691.880549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204691.880553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204691.880555:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204692.200486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204692.200492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204692.200494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204692.200499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204692.200502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204692.200505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204692.200508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204692.200510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204692.200512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204692.648465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204692.648470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204692.648472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204692.648475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204692.648478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204692.648480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204692.648482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204692.648483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204692.648485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204692.712465:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204692.712466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204692.712473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204692.712474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204692.712476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204692.712477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204692.712483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204692.712483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204692.712487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204692.712491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204692.712492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204692.712494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204692.712497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204692.712500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204692.712504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204692.712504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204692.712507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204692.712511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204692.904516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204692.904522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204692.904528:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204692.904529:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204692.904532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204692.904534:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204692.904536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204692.904538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204692.904555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204692.904557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204692.904560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204693.224471:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300819969, at 4300820480 00000100:00000040:1.0:1788204693.224471:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300819969, at 4300820480 00000100:00000001:2.0:1788204693.224473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204693.224479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788204693.224483:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:2.0:1788204693.224483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204693.224486:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00100000:2.0:1788204693.224489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204693.224494:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204693.224497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.224501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.224503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.224506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204693.672446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204693.672451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204693.672453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204693.672456:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204693.672459:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204693.672462:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204693.672463:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204693.672465:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204693.672468:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204693.736461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204693.736462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204693.736469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204693.736470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204693.736473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204693.736474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204693.736481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204693.736482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204693.736487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204693.736488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204693.736490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204693.736492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204693.736494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204693.736497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204693.736500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204693.736503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204693.736506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204693.736511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.736694:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204693.736699:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204693.736704:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204693.736715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.736731:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.736734:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.736742:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204693.736747:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204693.736754:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204693.736762:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205888128:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204693.736771:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204693.736783:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204693.736786:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.736792:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204693.736799:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204693.736804:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204693.736813:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204693.736819:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204693.736839:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204693.736845:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888128, portal 25 00000100:00000001:2.0:1788204693.736851:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204693.736855:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:2.0:1788204693.736867:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.736888:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204693.736895:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204693.736900:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204693.736906:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205888128, offset 0 00000400:00000200:2.0:1788204693.736918:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204693.736931:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.736941:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204693.736950:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204693.736958:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204693.736967:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204693.736974:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204693.736984:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58794:58794:256:4294967295] 192.168.201.151@tcp LPNI seq info [58774:58774:8:4294967295] 00000400:00000200:2.0:1788204693.736993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737008:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204693.737018:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737026:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204693.737034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204693.737040:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204693.737047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204693.737052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204693.737056:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737059:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204693.737066:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204693.737073:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204693.737078:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204693.737081:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204693.737087:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204693.737093:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737115:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.737123:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737139:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737144:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204693.737157:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204693.737164:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204693.737169:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300825089 00000100:00000010:2.0:1788204693.737177:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1cc0. 00000100:00000001:2.0:1788204693.737186:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204693.737195:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300825089 00000100:00000010:2.0:1788204693.737201:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c17e0. 00000100:00000001:2.0:1788204693.737207:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204693.737213:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204693.737216:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737218:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204693.737224:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204693/real 1788204693] req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204693.737239:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204693.737240:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204693.737242:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204693.737244:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204693.737246:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204693.737249:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204693.737252:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204693.737260:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737266:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204693.737268:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737270:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737272:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204693.737276:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737283:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204693.737287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737294:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204693.737297:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737299:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737300:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737302:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204693.737304:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204693.737306:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204693.737310:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204693.737313:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204693.737316:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204693.737325:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205888128:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204693.737329:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737331:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.737334:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205888128/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737340:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204693.737342:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204693.737344:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204693.737347:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204693.737348:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204693.737350:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204693.737352:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204693.737354:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:2.0:1788204693.737357:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204693.737359:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204693.737360:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:2.0:1788204693.737387:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204693.737389:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737391:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737397:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204693.737399:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737401:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204693.737402:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204693.737403:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204693.737407:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737414:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.737415:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737417:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204693.737419:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204693.737421:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204693.737424:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888256:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204693.737427:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204693.737429:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204693.737430:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737432:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204693.737435:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204693.737437:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204693.737440:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204693.737442:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204693.737444:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204693.737446:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888256, portal 10 00000100:00000001:2.0:1788204693.737448:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204693.737448:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:2.0:1788204693.737453:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737457:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204693.737458:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204693.737459:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204693.737461:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205888256, offset 0 00000400:00000200:2.0:1788204693.737463:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737468:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204693.737470:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204693.737472:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204693.737474:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204693.737475:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737477:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58795:58795:256:4294967295] 192.168.201.151@tcp LPNI seq info [58775:58775:8:4294967295] 00000400:00000200:2.0:1788204693.737479:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737483:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204693.737486:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737487:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204693.737490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204693.737492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204693.737495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204693.737496:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204693.737498:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737499:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204693.737509:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204693.737511:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204693.737513:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204693.737514:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204693.737519:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204693.737523:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737529:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.737532:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737537:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737539:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204693.737542:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204693.737545:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204693.737548:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204693.737550:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300821147 00000100:00000001:2.0:1788204693.737559:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.737562:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300821147 00000100:00000001:2.0:1788204693.737564:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.737566:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204693.737568:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737569:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204693.737573:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204693/real 1788204693] req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204693.737578:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204693.737579:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204693.737580:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204693.737582:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204693.737583:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204693.737585:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204693.737586:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204693.737591:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737595:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204693.737596:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737597:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737598:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204693.737600:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737603:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204693.737605:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737611:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204693.737613:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737614:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737615:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737617:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204693.737619:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204693.737622:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204693.737625:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204693.737627:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204693.737631:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204693.737638:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888256:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204693.737641:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204693.737645:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737651:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.737653:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737655:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204693.737656:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204693.737658:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204693.737661:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888384:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204693.737664:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204693.737666:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204693.737667:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737669:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204693.737672:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204693.737674:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204693.737678:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204693.737681:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204693.737684:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204693.737686:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888384, portal 4 00000100:00000001:2.0:1788204693.737688:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204693.737689:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:2.0:1788204693.737694:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737701:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204693.737703:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204693.737705:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204693.737707:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205888384, offset 0 00000400:00000200:2.0:1788204693.737711:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737715:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737718:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204693.737721:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204693.737724:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204693.737726:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204693.737728:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737731:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58796:58796:256:4294967295] 192.168.201.151@tcp LPNI seq info [58776:58776:8:4294967295] 00000400:00000200:2.0:1788204693.737735:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737741:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204693.737746:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737748:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204693.737751:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204693.737752:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204693.737755:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204693.737756:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204693.737758:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737759:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204693.737761:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204693.737764:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204693.737765:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204693.737766:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204693.737768:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204693.737770:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737775:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.737777:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737780:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737781:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204693.737784:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204693.737785:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204693.737787:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204693.737788:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300821033 00000100:00000001:2.0:1788204693.737790:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.737791:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300821033 00000100:00000001:2.0:1788204693.737793:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.737794:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204693.737796:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.737797:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204693.737799:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204693/real 1788204693] req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204693.737803:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204693.737804:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204693.737805:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204693.737806:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204693.737807:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204693.737808:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204693.737809:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204693.737814:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737818:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204693.737830:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737832:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737833:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204693.737836:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737840:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204693.737842:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737846:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204693.737847:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737848:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.737849:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.737850:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204693.737852:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204693.737854:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204693.737856:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204693.737857:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204693.737859:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204693.737863:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888384:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204693.737865:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204693.737867:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737871:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.737871:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737873:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204693.737874:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204693.737875:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204693.737876:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888512:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204693.737878:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204693.737879:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204693.737880:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204693.737881:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204693.737883:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204693.737884:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204693.737886:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204693.737888:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204693.737890:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204693.737891:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888512, portal 4 00000100:00000001:2.0:1788204693.737893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204693.737894:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:2.0:1788204693.737898:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.737904:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204693.737907:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204693.737908:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204693.737911:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205888512, offset 0 00000400:00000200:2.0:1788204693.737914:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737918:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737921:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204693.737924:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204693.737926:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204693.737928:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204693.737931:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204693.737934:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58797:58797:256:4294967295] 192.168.201.151@tcp LPNI seq info [58777:58777:8:4294967295] 00000400:00000200:2.0:1788204693.737937:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204693.737943:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204693.737947:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737950:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204693.737954:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204693.737957:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1788204693.737961:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204693.737963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1788204693.737979:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204693.737981:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204693.737983:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204693.737987:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204693.737990:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204693.737991:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204693.737994:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204693.737997:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738005:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.738008:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738019:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.738020:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204693.738023:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204693.738024:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204693.738027:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204693.738028:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300821039 00000100:00000001:2.0:1788204693.738030:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.738031:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300821039 00000100:00000001:2.0:1788204693.738033:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300825089 00000100:00000001:2.0:1788204693.738034:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204693.738035:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.738036:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204693.738039:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204693/real 1788204693] req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204693.738043:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204693.738044:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204693.738045:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204693.738046:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204693.738048:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204693.738049:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204693.738051:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204693.738055:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738059:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204693.738060:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738061:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.738062:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204693.738063:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738068:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204693.738070:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738074:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204693.738075:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738076:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204693.738077:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738078:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204693.738079:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204693.738081:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204693.738084:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204693.738086:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204693.738089:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204693.738096:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888512:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204693.738100:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738101:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.738104:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205888256/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738110:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204693.738112:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204693.738113:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204693.738116:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204693.738117:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204693.738119:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204693.738121:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204693.738122:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:2.0:1788204693.738125:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204693.738126:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204693.738128:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:2.0:1788204693.738131:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204693.738132:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738133:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.738136:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205888384/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738142:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204693.738143:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204693.738144:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204693.738146:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204693.738148:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204693.738149:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204693.738151:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204693.738152:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:2.0:1788204693.738154:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204693.738155:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204693.738156:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:2.0:1788204693.738159:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204693.738160:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738161:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204693.738164:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205888512/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204693.738170:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204693.738171:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204693.738172:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204693.738174:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204693.738175:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204693.738176:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204693.738178:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204693.738180:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:2.0:1788204693.738181:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204693.738182:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204693.738184:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:2.0:1788204693.738185:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204693.738187:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738189:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204693.738192:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204693.738194:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.738196:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204693.738197:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204693.928476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204693.928482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204693.928486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204693.928487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204693.928489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204693.928490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204693.928492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204693.928493:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204693.928559:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204693.928563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204693.928566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204694.248489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204694.248495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204694.248497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204694.248508:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204694.248515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204694.248519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204694.248521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204694.248524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204694.248527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204694.696478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204694.696485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204694.696487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204694.696492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204694.696495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204694.696498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204694.696500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204694.696502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204694.696503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204694.760468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204694.760468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204694.760475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204694.760475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204694.760477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204694.760478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204694.760484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204694.760485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204694.760490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204694.760490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204694.760495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204694.760495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204694.760497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204694.760500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204694.760502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204694.760504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204694.760506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204694.760508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204694.952903:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204694.952907:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204694.952911:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204694.952912:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204694.952914:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204694.952914:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204694.952917:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204694.952917:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204694.952929:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204694.952932:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204694.952934:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204695.272456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204695.272461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204695.272463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.272466:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204695.272469:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204695.272472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204695.272474:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204695.272475:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204695.272477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204695.720501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204695.720512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204695.720514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204695.720520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204695.720524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204695.720528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204695.720530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204695.720532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204695.720535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204695.784543:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204695.784550:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204695.784551:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204695.784556:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204695.784561:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204695.784561:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204695.784574:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204695.784575:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204695.784583:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204695.784584:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204695.784587:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204695.784592:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204695.784593:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204695.784599:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204695.784603:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204695.784606:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204695.784609:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204695.784613:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204695.976492:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204695.976502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204695.976513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204695.976514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204695.976517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204695.976519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204695.976522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204695.976523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204695.976641:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204695.976645:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204695.976649:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204696.296480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204696.296486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204696.296488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204696.296493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204696.296496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204696.296499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204696.296502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204696.296505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204696.296507:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204696.744507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204696.744516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204696.744518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204696.744523:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204696.744526:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204696.744529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204696.744531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204696.744533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204696.744536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204696.808497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204696.808497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204696.808505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204696.808508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204696.808514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788204696.808515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204696.808518:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204696.808522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204696.808523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204696.808526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204696.808528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204696.808529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204696.808535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204696.808538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204696.808541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204696.808545:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204696.808545:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204696.808557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204697.000559:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204697.000569:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204697.000576:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204697.000577:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204697.000581:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204697.000582:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204697.000585:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204697.000586:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204697.000609:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204697.000613:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204697.000616:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204697.320438:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204697.320448:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204697.320450:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204697.320457:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204697.320462:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204697.320468:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204697.320472:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204697.320474:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204697.320478:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204697.768469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204697.768478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204697.768480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204697.768486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204697.768491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204697.768495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204697.768497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204697.768500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204697.768503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204697.832462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204697.832472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204697.832476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204697.832481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204697.832488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204697.832492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204697.832504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204697.832507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204697.832510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204698.024478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204698.024488:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204698.024495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204698.024496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204698.024499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204698.024501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204698.024504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204698.024505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204698.024578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204698.024583:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204698.024586:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204698.344499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204698.344500:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300825089, at 4300825600 00000100:00000001:2.0:1788204698.344508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204698.344512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204698.344518:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204698.344519:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000001:2.0:1788204698.344524:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204698.344529:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.344533:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204698.344536:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.344540:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204698.344624:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300825089, at 4300825600 00000100:00000010:0.0:1788204698.344632:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000001:1.0:1788204698.792548:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204698.792558:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204698.792561:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204698.792568:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204698.792572:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204698.792577:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204698.792580:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204698.792583:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204698.792587:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204698.856500:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204698.856506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204698.856509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204698.856516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204698.856522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204698.856529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204698.856533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204698.856539:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204698.856544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.856795:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204698.856800:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204698.856803:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204698.856812:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.856826:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204698.856829:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.856833:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204698.856836:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204698.856840:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204698.856847:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205888640:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204698.856854:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204698.856859:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204698.856863:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.856869:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204698.856876:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204698.856881:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204698.856891:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204698.856896:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204698.856907:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204698.856918:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888640, portal 25 00000100:00000001:2.0:1788204698.856924:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204698.856928:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300822400 : -113511408729216 : ffff98c310ce0380) 00000100:00000040:2.0:1788204698.856938:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.856957:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204698.856964:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204698.856969:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204698.856975:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205888640, offset 0 00000400:00000200:2.0:1788204698.856990:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857006:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857016:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204698.857022:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204698.857027:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204698.857031:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204698.857034:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857038:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58798:58798:256:4294967295] 192.168.201.151@tcp LPNI seq info [58778:58778:8:4294967295] 00000400:00000200:2.0:1788204698.857041:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857053:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204698.857062:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857067:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204698.857082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204698.857088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204698.857098:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204698.857105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204698.857116:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857120:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204698.857129:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204698.857139:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204698.857149:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204698.857153:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204698.857161:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204698.857168:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857185:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.857193:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857206:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857210:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204698.857219:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204698.857228:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204698.857233:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300830209 00000100:00000010:2.0:1788204698.857241:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c15a0. 00000100:00000001:2.0:1788204698.857252:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204698.857260:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300830209 00000100:00000010:2.0:1788204698.857266:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a20. 00000100:00000001:2.0:1788204698.857272:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204698.857279:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204698.857284:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857290:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204698.857300:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204698/real 1788204698] req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204698.857320:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204698.857324:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204698.857328:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204698.857331:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204698.857334:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204698.857339:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204698.857345:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204698.857403:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857422:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204698.857427:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857432:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.857436:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204698.857444:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857455:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204698.857461:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857469:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204698.857472:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857474:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.857475:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857478:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204698.857480:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204698.857483:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204698.857487:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204698.857491:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204698.857496:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204698.857502:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205888640:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204698.857507:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857508:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788204698.857510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:2.0:1788204698.857511:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0380 x1875062205888640/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204698.857514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204698.857517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857521:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1788204698.857522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204698.857525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788204698.857525:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1788204698.857526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788204698.857528:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 00000100:00000001:1.0:1788204698.857529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204698.857532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204698.857533:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1788204698.857534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204698.857536:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204698.857538:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204698.857540:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204698.857543:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:2.0:1788204698.857546:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204698.857548:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204698.857550:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0380. 00000100:00000001:2.0:1788204698.857552:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204698.857553:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857554:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857561:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204698.857562:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857563:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204698.857564:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204698.857565:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204698.857567:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857572:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204698.857573:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.857574:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204698.857576:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204698.857577:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204698.857579:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888768:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204698.857581:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204698.857582:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204698.857583:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.857584:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204698.857586:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204698.857587:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204698.857590:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204698.857592:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204698.857595:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204698.857597:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888768, portal 10 00000100:00000001:2.0:1788204698.857598:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204698.857599:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:2.0:1788204698.857604:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857618:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204698.857621:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204698.857623:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204698.857625:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205888768, offset 0 00000400:00000200:2.0:1788204698.857628:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857631:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857633:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204698.857635:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204698.857637:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204698.857639:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204698.857640:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857642:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58799:58799:256:4294967295] 192.168.201.151@tcp LPNI seq info [58779:58779:8:4294967295] 00000400:00000200:2.0:1788204698.857645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857649:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204698.857651:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857653:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204698.857656:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204698.857658:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204698.857662:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204698.857665:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204698.857674:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857676:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204698.857679:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204698.857683:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204698.857686:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204698.857687:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204698.857690:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204698.857694:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857702:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.857705:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857717:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857718:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204698.857723:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204698.857728:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204698.857742:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204698.857744:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300826267 00000100:00000001:2.0:1788204698.857747:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.857750:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300826267 00000100:00000001:2.0:1788204698.857751:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.857753:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204698.857754:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857756:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204698.857759:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204698/real 1788204698] req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204698.857764:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204698.857765:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204698.857767:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204698.857768:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204698.857769:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204698.857771:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204698.857773:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204698.857778:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857782:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204698.857783:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857784:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.857784:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204698.857786:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857790:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204698.857792:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857797:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204698.857798:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857799:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.857800:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.857801:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204698.857803:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204698.857805:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204698.857808:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204698.857809:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204698.857813:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204698.857819:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888768:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204698.857822:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204698.857824:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857828:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204698.857829:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.857830:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204698.857832:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204698.857833:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204698.857834:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888896:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204698.857837:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204698.857838:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204698.857840:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.857841:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204698.857845:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204698.857846:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204698.857850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204698.857853:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204698.857856:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204698.857859:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205888896, portal 4 00000100:00000001:2.0:1788204698.857861:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204698.857862:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:2.0:1788204698.857868:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857875:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204698.857877:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204698.857879:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204698.857882:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205888896, offset 0 00000400:00000200:2.0:1788204698.857885:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857887:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857890:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204698.857892:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204698.857894:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204698.857897:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204698.857899:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204698.857902:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58800:58800:256:4294967295] 192.168.201.151@tcp LPNI seq info [58780:58780:8:4294967295] 00000400:00000200:2.0:1788204698.857906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.857911:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204698.857916:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857919:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204698.857923:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204698.857926:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204698.857930:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204698.857933:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204698.857937:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.857938:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204698.857941:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204698.857945:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204698.857948:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204698.857949:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204698.857952:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204698.857956:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857963:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.857966:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.857973:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857974:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204698.857978:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204698.857981:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204698.857984:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204698.857986:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300826153 00000100:00000001:2.0:1788204698.857988:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.857990:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300826153 00000100:00000001:2.0:1788204698.857992:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.857994:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204698.857997:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.857999:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204698.858002:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204698/real 1788204698] req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204698.858009:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204698.858011:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204698.858012:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204698.858014:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204698.858016:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204698.858018:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204698.858022:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204698.858030:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858042:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204698.858044:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858046:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.858047:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204698.858051:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858057:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204698.858061:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858068:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204698.858070:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858072:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.858073:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858075:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204698.858077:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204698.858080:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204698.858083:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204698.858085:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204698.858089:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204698.858097:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205888896:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204698.858101:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204698.858104:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858109:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204698.858110:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.858112:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204698.858114:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204698.858115:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204698.858118:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204698.858121:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204698.858123:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204698.858124:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204698.858126:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204698.858129:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204698.858132:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204698.858135:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204698.858137:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204698.858141:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204698.858143:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205889024, portal 4 00000100:00000001:2.0:1788204698.858146:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204698.858147:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:2.0:1788204698.858152:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858159:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204698.858161:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204698.858163:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204698.858165:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205889024, offset 0 00000400:00000200:2.0:1788204698.858169:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204698.858172:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.858175:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204698.858178:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204698.858181:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204698.858183:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204698.858185:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204698.858188:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58801:58801:256:4294967295] 192.168.201.151@tcp LPNI seq info [58781:58781:8:4294967295] 00000400:00000200:2.0:1788204698.858192:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204698.858197:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204698.858201:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.858205:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204698.858209:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204698.858212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204698.858216:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204698.858219:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204698.858223:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204698.858225:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204698.858228:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204698.858232:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204698.858234:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204698.858236:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204698.858239:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204698.858242:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858250:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.858253:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858259:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.858261:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204698.858264:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204698.858267:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204698.858270:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204698.858272:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300826159 00000100:00000001:2.0:1788204698.858274:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.858276:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300826159 00000100:00000001:2.0:1788204698.858278:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300830209 00000100:00000001:2.0:1788204698.858280:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204698.858282:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.858283:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204698.858287:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204698/real 1788204698] req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204698.858294:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204698.858295:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204698.858297:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204698.858299:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204698.858301:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204698.858303:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204698.858306:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204698.858315:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858322:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204698.858324:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858326:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.858327:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204698.858330:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858336:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204698.858340:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858347:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204698.858349:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858350:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204698.858352:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858354:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204698.858356:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204698.858359:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204698.858362:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204698.858381:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204698.858386:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204698.858394:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204698.858397:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858399:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.858402:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205888768/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858409:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204698.858410:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204698.858412:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204698.858415:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204698.858416:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204698.858418:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204698.858420:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204698.858422:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:2.0:1788204698.858424:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204698.858425:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204698.858426:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:2.0:1788204698.858429:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204698.858430:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858432:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.858436:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205888896/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858442:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204698.858444:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204698.858445:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204698.858448:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204698.858449:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204698.858450:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204698.858452:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204698.858453:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:2.0:1788204698.858456:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204698.858457:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204698.858458:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:2.0:1788204698.858461:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204698.858462:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858464:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204698.858467:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205889024/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204698.858473:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204698.858474:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204698.858476:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204698.858478:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204698.858479:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204698.858480:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204698.858482:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204698.858483:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:2.0:1788204698.858486:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204698.858487:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204698.858488:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:2.0:1788204698.858490:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204698.858491:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858493:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.858498:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204698.858499:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.858501:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204698.858502:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048543:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204699.048553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204699.048563:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204699.048567:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204699.048571:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048574:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204699.048576:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048578:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048579:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048583:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204699.048584:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204699.048586:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204699.048587:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204699.048603:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204699.048606:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204699.048608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204699.368504:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204699.368514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204699.368516:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204699.368522:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204699.368526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204699.368530:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204699.368534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204699.368537:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204699.368541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204699.816492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204699.816499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204699.816501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204699.816506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204699.816509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204699.816513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204699.816515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204699.816517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204699.816519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204699.880481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204699.880481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204699.880488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204699.880488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204699.880492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204699.880493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204699.880500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204699.880501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204699.880507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204699.880507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204699.880513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204699.880514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204699.880520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204699.880522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204699.880525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204699.880527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204699.880530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204699.880533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204700.072489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204700.072495:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204700.072501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204700.072503:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204700.072506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204700.072507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204700.072509:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204700.072511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204700.072576:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204700.072580:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204700.072583:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204700.392488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204700.392497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204700.392499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204700.392506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204700.392511:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204700.392515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204700.392519:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204700.392521:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204700.392524:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204700.840469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204700.840475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204700.840478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204700.840483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204700.840486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204700.840490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204700.840492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204700.840493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204700.840495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204700.904469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204700.904473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204700.904475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204700.904478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204700.904479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204700.904482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204700.904484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204700.904487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204700.904489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204700.904492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204700.904494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204700.904497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204700.904497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204700.904500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204700.904504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204700.904507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204700.904510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204700.904513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096594:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204701.096600:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204701.096603:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204701.096605:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096608:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204701.096610:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096611:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096612:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096614:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204701.096615:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096617:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204701.096618:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204701.096619:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204701.096620:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096621:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204701.096622:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096623:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096625:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096628:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204701.096629:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204701.096630:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204701.096631:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096633:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204701.096634:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096635:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204701.096637:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204701.096654:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204701.096657:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204701.096659:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204701.416474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204701.416482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204701.416484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204701.416489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204701.416492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204701.416496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204701.416498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204701.416502:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204701.416505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204701.864496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204701.864504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204701.864506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204701.864512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204701.864515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204701.864518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204701.864520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204701.864522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204701.864525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204701.928483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204701.928483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204701.928491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204701.928492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204701.928494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204701.928495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204701.928503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204701.928503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204701.928510:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204701.928510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204701.928517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204701.928517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204701.928525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204701.928529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204701.928533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204701.928534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204701.928537:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204701.928542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204702.120503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204702.120511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204702.120519:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204702.120521:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204702.120525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204702.120527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204702.120530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204702.120532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204702.120591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204702.120597:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204702.120600:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204702.440455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204702.440467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204702.440469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204702.440476:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204702.440480:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204702.440485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204702.440487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204702.440490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204702.440494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204702.888545:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204702.888555:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204702.888558:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204702.888566:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204702.888571:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204702.888575:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204702.888579:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204702.888582:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204702.888586:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204702.952509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204702.952509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204702.952520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204702.952520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204702.952524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204702.952524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204702.952534:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204702.952534:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204702.952541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204702.952541:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204702.952548:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204702.952548:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204702.952554:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204702.952558:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204702.952562:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204702.952563:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204702.952567:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204702.952570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204703.144539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204703.144549:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204703.144555:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204703.144557:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204703.144561:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204703.144562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204703.144565:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204703.144567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204703.144589:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204703.144592:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204703.144596:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204703.464460:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300830209, at 4300830720 00000100:00000040:1.0:1788204703.464460:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300830209, at 4300830720 00000100:00000001:2.0:1788204703.464470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204703.464477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788204703.464478:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000001:2.0:1788204703.464480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204703.464481:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00100000:2.0:1788204703.464484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204703.464489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204703.464493:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.464496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204703.464500:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.464503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204703.912465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204703.912471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204703.912472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204703.912476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204703.912479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204703.912482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204703.912484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204703.912485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204703.912488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204703.976455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204703.976455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204703.976461:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204703.976462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204703.976465:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204703.976465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204703.976472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204703.976472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204703.976477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204703.976478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204703.976480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204703.976483:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204703.976483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204703.976487:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204703.976490:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204703.976494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204703.976498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204703.976501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.976733:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204703.976747:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204703.976755:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204703.976773:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.976808:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204703.976814:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.976824:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204703.976830:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204703.976838:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204703.976850:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205889152:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204703.976858:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204703.976864:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204703.976868:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.976872:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204703.976881:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204703.976885:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204703.976901:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204703.976906:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204703.976917:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204703.976926:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205889152, portal 25 00000100:00000001:2.0:1788204703.976930:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204703.976934:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:2.0:1788204703.976943:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.976971:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204703.976979:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204703.976985:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204703.976993:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205889152, offset 0 00000400:00000200:2.0:1788204703.977014:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204703.977036:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.977060:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204703.977076:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204703.977090:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204703.977104:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204703.977113:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204703.977125:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58802:58802:256:4294967295] 192.168.201.151@tcp LPNI seq info [58782:58782:8:4294967295] 00000400:00000200:2.0:1788204703.977136:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.977164:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204703.977189:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.977202:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204703.977219:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204703.977229:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204703.977246:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204703.977251:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204703.977258:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.977264:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204703.977276:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204703.977288:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204703.977298:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204703.977304:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204703.977314:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204703.977324:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977347:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.977355:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977408:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.977412:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204703.977420:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204703.977429:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204703.977432:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300835330 00000100:00000010:2.0:1788204703.977437:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ea0. 00000100:00000001:2.0:1788204703.977444:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204703.977453:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300835330 00000100:00000010:2.0:1788204703.977456:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c00. 00000100:00000001:2.0:1788204703.977459:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204703.977462:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204703.977466:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.977468:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204703.977475:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204703/real 1788204703] req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204703.977491:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204703.977492:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204703.977494:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204703.977497:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204703.977498:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204703.977501:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204703.977505:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204703.977512:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977516:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204703.977518:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977520:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.977521:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204703.977524:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977529:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204703.977533:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977537:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204703.977538:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977539:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.977540:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977542:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204703.977544:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204703.977547:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204703.977551:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204703.977553:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204703.977557:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204703.977565:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205889152:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204703.977570:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977572:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.977575:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205889152/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977583:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204703.977585:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204703.977588:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204703.977591:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204703.977594:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204703.977596:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204703.977598:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204703.977601:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:2.0:1788204703.977605:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204703.977607:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204703.977609:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:2.0:1788204703.977613:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204703.977614:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977617:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977624:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204703.977626:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.977628:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204703.977630:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204703.977631:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204703.977635:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977652:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204703.977654:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.977656:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204703.977658:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204703.977660:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204703.977663:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889280:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204703.977667:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204703.977669:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204703.977671:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.977673:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204703.977676:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204703.977678:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204703.977682:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204703.977685:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204703.977688:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204703.977691:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205889280, portal 10 00000100:00000001:2.0:1788204703.977693:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204703.977694:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:2.0:1788204703.977701:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977707:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204703.977710:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204703.977712:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204703.977715:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205889280, offset 0 00000400:00000200:2.0:1788204703.977719:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204703.977724:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.977728:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204703.977731:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204703.977734:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204703.977735:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204703.977737:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204703.977739:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58803:58803:256:4294967295] 192.168.201.151@tcp LPNI seq info [58783:58783:8:4294967295] 00000400:00000200:2.0:1788204703.977742:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.977746:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204703.977749:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.977752:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204703.977754:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204703.977757:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204703.977761:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204703.977762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204703.977765:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.977766:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204703.977768:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204703.977770:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204703.977772:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204703.977773:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204703.977775:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204703.977777:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977784:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.977787:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977793:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.977794:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204703.977797:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204703.977802:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204703.977805:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204703.977807:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300831387 00000100:00000001:2.0:1788204703.977810:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.977812:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300831387 00000100:00000001:2.0:1788204703.977814:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.977816:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204703.977818:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.977820:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204703.977824:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204703/real 1788204703] req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204703.977832:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204703.977834:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204703.977836:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204703.977838:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204703.977840:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204703.977843:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204703.977846:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204703.977853:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977860:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204703.977861:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977863:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.977864:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204703.977868:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977874:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204703.977878:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977884:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204703.977886:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977888:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.977889:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.977891:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204703.977893:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204703.977896:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204703.977899:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204703.977901:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204703.977906:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204703.977913:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889280:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204703.977918:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204703.977922:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977928:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204703.977930:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.977932:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204703.977934:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204703.977936:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204703.977938:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889408:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204703.977941:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204703.977944:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204703.977945:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.977947:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204703.977950:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204703.977952:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204703.977956:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204703.977958:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204703.977962:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204703.977965:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205889408, portal 4 00000100:00000001:2.0:1788204703.977966:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204703.977968:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:2.0:1788204703.977973:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.977980:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204703.977982:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204703.977985:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204703.977987:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205889408, offset 0 00000400:00000200:2.0:1788204703.977991:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204703.977995:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.977998:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204703.978001:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204703.978004:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204703.978006:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204703.978009:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204703.978011:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58804:58804:256:4294967295] 192.168.201.151@tcp LPNI seq info [58784:58784:8:4294967295] 00000400:00000200:2.0:1788204703.978015:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.978021:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204703.978026:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.978029:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204703.978033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204703.978036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204703.978040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204703.978043:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204703.978047:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.978049:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204703.978052:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204703.978056:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204703.978059:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204703.978060:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204703.978063:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204703.978067:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978074:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.978077:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978083:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.978085:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204703.978088:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204703.978092:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204703.978095:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204703.978097:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300831273 00000100:00000001:2.0:1788204703.978099:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.978101:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300831273 00000100:00000001:2.0:1788204703.978104:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.978106:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204703.978108:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.978110:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204703.978114:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204703/real 1788204703] req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204703.978122:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204703.978123:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204703.978125:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204703.978127:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204703.978129:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204703.978131:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204703.978134:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204703.978142:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978149:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204703.978150:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978153:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.978154:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204703.978156:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978161:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204703.978163:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978167:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204703.978169:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978170:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.978170:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978172:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204703.978174:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204703.978176:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204703.978178:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204703.978180:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204703.978182:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204703.978186:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889408:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204703.978189:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204703.978191:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978195:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204703.978196:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.978197:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204703.978199:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204703.978200:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204703.978202:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889536:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204703.978204:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204703.978205:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204703.978206:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204703.978207:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204703.978211:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204703.978212:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204703.978214:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204703.978216:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204703.978218:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204703.978221:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205889536, portal 4 00000100:00000001:2.0:1788204703.978222:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204703.978223:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000040:2.0:1788204703.978228:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978235:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204703.978237:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204703.978239:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204703.978242:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205889536, offset 0 00000400:00000200:2.0:1788204703.978245:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204703.978249:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.978252:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204703.978255:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204703.978257:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204703.978260:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204703.978262:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204703.978265:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58805:58805:256:4294967295] 192.168.201.151@tcp LPNI seq info [58785:58785:8:4294967295] 00000400:00000200:2.0:1788204703.978269:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204703.978275:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204703.978279:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.978282:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204703.978286:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204703.978289:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204703.978292:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204703.978293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204703.978296:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204703.978297:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204703.978299:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204703.978301:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204703.978303:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204703.978305:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204703.978306:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204703.978308:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978313:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.978315:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978323:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.978324:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204703.978326:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204703.978328:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204703.978330:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204703.978332:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300831279 00000100:00000001:2.0:1788204703.978333:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.978335:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300831279 00000100:00000001:2.0:1788204703.978336:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300835330 00000100:00000001:2.0:1788204703.978337:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204703.978338:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.978339:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204703.978341:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204703/real 1788204703] req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204703.978345:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204703.978346:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204703.978347:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204703.978349:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204703.978350:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204703.978351:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204703.978353:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204703.978358:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978361:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204703.978362:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978362:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.978379:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204703.978381:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978385:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204703.978388:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978392:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204703.978393:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978394:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204703.978395:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978395:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204703.978397:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204703.978398:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204703.978400:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204703.978401:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204703.978403:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204703.978407:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889536:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204703.978409:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978410:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.978412:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205889280/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978416:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204703.978417:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204703.978418:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204703.978420:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204703.978420:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204703.978421:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204703.978423:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204703.978424:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:2.0:1788204703.978426:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204703.978427:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204703.978428:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:2.0:1788204703.978430:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204703.978430:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978431:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.978433:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205889408/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978437:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204703.978438:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204703.978438:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204703.978439:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204703.978440:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204703.978440:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204703.978441:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204703.978442:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:2.0:1788204703.978444:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204703.978444:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204703.978445:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:2.0:1788204703.978447:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204703.978448:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978450:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204703.978452:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205889536/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204703.978458:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204703.978459:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204703.978461:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204703.978463:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204703.978465:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204703.978466:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204703.978468:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204703.978470:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:2.0:1788204703.978472:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204703.978473:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204703.978474:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204703.978477:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204703.978478:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978480:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204703.978483:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204703.978485:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.978487:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204703.978488:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204704.168506:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204704.168515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204704.168524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204704.168526:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204704.168530:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204704.168532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204704.168535:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204704.168536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204704.168623:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204704.168629:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204704.168634:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204704.488467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204704.488477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204704.488480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204704.488487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204704.488491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204704.488496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204704.488499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204704.488503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204704.488506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204704.936445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204704.936450:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204704.936451:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204704.936454:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204704.936457:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204704.936460:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204704.936461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204704.936463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204704.936465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204705.000481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204705.000481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204705.000489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204705.000489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204705.000492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204705.000493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204705.000499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204705.000500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204705.000504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204705.000506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204705.000507:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204705.000510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204705.000510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204705.000513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204705.000516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204705.000520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204705.000522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204705.000525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204705.192529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204705.192536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204705.192542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204705.192543:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204705.192546:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204705.192548:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204705.192551:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204705.192552:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204705.192571:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204705.192573:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204705.192576:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204705.512501:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204705.512511:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204705.512514:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204705.512522:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204705.512527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204705.512532:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204705.512536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204705.512539:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204705.512542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204705.960475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204705.960480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204705.960482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204705.960485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204705.960488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204705.960491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204705.960492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204705.960494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204705.960496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204706.024462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204706.024465:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204706.024472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204706.024472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204706.024476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204706.024477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204706.024484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204706.024485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204706.024490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204706.024491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204706.024495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204706.024495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204706.024502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204706.024505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204706.024509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204706.024510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204706.024513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204706.024518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204706.216472:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204706.216479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204706.216485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204706.216487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204706.216490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204706.216492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204706.216495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204706.216496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204706.216551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204706.216554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204706.216556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204706.536449:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204706.536455:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204706.536457:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204706.536462:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204706.536465:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204706.536469:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204706.536472:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204706.536474:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204706.536476:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204706.984491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204706.984498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204706.984500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204706.984505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204706.984509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204706.984512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204706.984514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204706.984517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204706.984520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204707.048476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204707.048477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204707.048485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204707.048485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204707.048489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204707.048489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204707.048497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204707.048497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204707.048503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204707.048504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204707.048509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204707.048510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204707.048515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204707.048519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204707.048523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204707.048523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204707.048527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204707.048531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204707.240523:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204707.240528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204707.240532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204707.240533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204707.240534:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204707.240535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204707.240537:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204707.240538:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204707.240554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204707.240555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204707.240557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204707.560482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204707.560490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204707.560492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204707.560497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204707.560500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204707.560504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204707.560506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204707.560508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204707.560510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204708.008519:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204708.008530:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204708.008533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204708.008539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204708.008544:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204708.008555:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204708.008558:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204708.008561:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204708.008565:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204708.072467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204708.072467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204708.072475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204708.072476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204708.072478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204708.072479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204708.072484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204708.072485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204708.072488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204708.072490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204708.072492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204708.072495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204708.072495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204708.072499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204708.072502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204708.072507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204708.072510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204708.072513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204708.264485:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204708.264492:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204708.264497:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204708.264497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204708.264500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204708.264501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204708.264504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204708.264505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204708.264563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204708.264578:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204708.264581:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204708.584497:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300835330, at 4300835840 00000100:00000040:1.0:1788204708.584497:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300835330, at 4300835840 00000100:00000001:2.0:1788204708.584497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204708.584506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204708.584511:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204708.584519:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000010:1.0:1788204708.584520:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00100000:2.0:1788204708.584520:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204708.584531:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204708.584535:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204708.584538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204708.584539:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204708.584541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204709.032467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204709.032475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204709.032477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204709.032483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204709.032486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204709.032490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204709.032492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204709.032494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204709.032497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204709.096471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204709.096471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204709.096477:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204709.096479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204709.096480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204709.096484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204709.096485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204709.096485:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204709.096489:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204709.096492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204709.096492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204709.096492:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204709.096499:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204709.096500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204709.096500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204709.096505:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6163 00000100:00000001:0.0:1788204709.096507:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204709.096507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204709.096513:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204709.096514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204709.096519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204709.096520:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204709.096523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204709.096524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204709.096524:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204709.096527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204709.096531:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204709.096533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096542:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096544:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096546:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204709.096550:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204709.096551:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204709.096576:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204709.096579:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204709.096580:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204709.096582:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204709.096584:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096586:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096587:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096589:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204709.096591:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204709.096593:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096594:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204709.096596:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204709.096599:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204709.096600:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096602:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204709.096603:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204709.096610:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204709.096613:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204709.096616:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204709.096617:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096619:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204709.096621:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204709.096624:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204709.096630:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204709.096635:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204709.096641:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204709.096648:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204709.096651:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204709.096654:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204709.096670:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204709.096673:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204709.096676:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204709.096677:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204709.096679:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204709.096681:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204709.096685:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6163 00000400:00000200:2.0:1788204709.096690:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204709.096695:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204709.096699:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204709.096706:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096711:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096716:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096722:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204709.096726:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204709.096728:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204709.096730:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204709.096734:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204709.096739:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204709.096741:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204709.096746:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096749:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096752:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096755:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204709.096757:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204709.096761:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096766:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204709.096768:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204709.096773:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204709.096776:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096779:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204709.096784:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204709.096787:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096788:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204709.096791:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204709.096795:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204709.096798:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204709.096800:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096804:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204709.096806:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204709.096809:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204709.096813:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204709.096817:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204709.096822:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204709.096828:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204709.096832:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204709.096836:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204709.096846:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204709.096849:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204709.096856:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204709.096858:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204709.096861:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204709.096865:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204709.096869:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6163 00000400:00000200:2.0:1788204709.096874:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204709.096876:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204709.096877:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204709.096879:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096880:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096883:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096885:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204709.096888:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204709.096889:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204709.096892:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204709.096894:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204709.096897:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204709.096901:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204709.096904:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096905:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.096908:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096910:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204709.096914:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204709.096916:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204709.096917:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204709.096918:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204709.096920:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204709.096922:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096926:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204709.096928:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204709.096929:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096932:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204709.096938:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204709.096940:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204709.096943:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204709.096945:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.096948:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204709.096950:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204709.096954:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204709.096959:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204709.096963:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204709.096980:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204709.096994:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204709.096997:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204709.097005:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204709.097015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204709.097023:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204709.097025:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204709.097028:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204709.097029:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204709.097030:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204709.097035:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204709.097039:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6163 00000400:00000200:2.0:1788204709.097045:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204709.097050:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204709.097054:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204709.097060:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.097064:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.097070:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.097074:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204709.097080:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204709.097083:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204709.097088:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204709.097092:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204709.097096:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204709.097099:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204709.097103:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.097107:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204709.097111:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204709.097115:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204709.097119:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204709.097124:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204709.097127:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204709.097131:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204709.097135:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204709.097145:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.097147:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204709.097151:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204709.097155:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.097158:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204709.097164:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204709.097167:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204709.097172:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204709.097175:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204709.097179:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204709.097182:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204709.097186:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204709.097193:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204709.097198:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204709.097207:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204709.097223:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204709.097227:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204709.097236:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204709.097257:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204709.097264:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6174) 00000800:00000200:3.1:1788204709.097297:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204709.097455:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204709.097600:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00010000:00000001:1.0:1788204709.288470:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204709.288475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288478:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204709.288480:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204709.288482:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204709.288484:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288486:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204709.288488:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288490:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288493:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288496:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204709.288497:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204709.288500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204709.288501:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204709.288518:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204709.288521:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204709.288524:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204710.056471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204710.056476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204710.056479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204710.056484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204710.056487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204710.056490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204710.056492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204710.056495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204710.056497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204710.120471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204710.120478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204710.120481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204710.120486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204710.120490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204710.120494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204710.120497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204710.120500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204710.120503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204710.120508:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204710.120512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204710.120513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204710.120516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204710.120517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204710.120519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204710.120520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204710.120521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204710.120523:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204710.312467:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204710.312472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204710.312475:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204710.312477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204710.312480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204710.312481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204710.312483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204710.312484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204710.312564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204710.312567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204710.312570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204711.080505:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204711.080515:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204711.080518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204711.080539:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204711.080543:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204711.080548:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204711.080551:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204711.080554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204711.080558:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204711.144474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204711.144487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204711.144491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204711.144500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204711.144506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204711.144513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204711.144532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204711.144537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204711.144543:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204711.144543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204711.144549:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204711.144551:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204711.144555:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204711.144558:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204711.144561:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204711.144564:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204711.144567:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204711.144570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204711.336500:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204711.336502:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204711.336505:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336510:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204711.336512:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336514:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336525:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336529:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204711.336530:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336533:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204711.336535:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204711.336536:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204711.336537:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336539:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204711.336540:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336542:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336543:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336547:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204711.336549:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204711.336550:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204711.336551:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336553:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204711.336554:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336556:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204711.336557:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204711.336572:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204711.336575:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204711.336577:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204712.104470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204712.104477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204712.104479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204712.104484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204712.104488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204712.104491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204712.104493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204712.104496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204712.104499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204712.168461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204712.168467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204712.168468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204712.168472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204712.168474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204712.168476:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204712.168478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204712.168480:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204712.168483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204712.168655:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204712.168658:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204712.168660:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204712.168663:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204712.168666:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204712.168668:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204712.168671:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204712.168672:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204712.168675:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.168868:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204712.168873:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204712.168879:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204712.168891:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204712] req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204712.168915:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204712.168919:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204712.168924:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5b00. 00000400:00000200:2.0:1788204712.168931:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4d48 00000400:00000010:2.0:1788204712.168935:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4d48. 00000100:00000001:2.0:1788204712.168941:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204712.168950:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204712.168984:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.168998:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204712.169002:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169007:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169010:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204712.169021:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169037:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204712.169046:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169068:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204712.169073:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169078:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169081:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169086:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204712.169091:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204712.169097:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204712.169104:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204712.169108:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204712.169112:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204712.169119:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205889664:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204712.169123:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.169125:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204712.169128:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205889664/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169138:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204712.169142:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204712.169145:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204712.169149:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204712.169152:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204712.169154:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204712.169156:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204712.169158:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204712.169161:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204712.169162:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204712.169164:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:2.0:1788204712.169168:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204712.169169:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169171:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.169173:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204712.169174:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204712.169175:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204712.169179:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204712] req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204712.169186:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204712.169187:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204712.169189:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f53c8. 00000400:00000200:2.0:1788204712.169192:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4660 00000400:00000010:2.0:1788204712.169193:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4660. 00000100:00000001:2.0:1788204712.169195:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204712.169198:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204712.169204:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169210:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204712.169211:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169213:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169214:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204712.169218:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169223:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204712.169226:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169232:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204712.169234:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169235:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169236:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169238:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204712.169240:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204712.169243:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204712.169246:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204712.169248:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204712.169252:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204712.169259:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205890048:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204712.169263:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204712.169266:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204712] req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204712.169272:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204712.169273:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204712.169275:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5058. 00000400:00000200:2.0:1788204712.169278:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4220 00000400:00000010:2.0:1788204712.169279:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4220. 00000100:00000001:2.0:1788204712.169281:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204712.169284:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204712.169302:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169308:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204712.169309:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169310:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169311:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204712.169314:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169320:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204712.169324:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169330:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204712.169332:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169334:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169335:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169336:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204712.169338:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204712.169340:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204712.169344:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204712.169346:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204712.169348:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204712.169353:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889792:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204712.169355:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204712.169358:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204712] req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204712.169391:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204712.169393:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204712.169396:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5160. 00000400:00000200:2.0:1788204712.169399:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4aa0 00000400:00000010:2.0:1788204712.169400:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4aa0. 00000100:00000001:2.0:1788204712.169402:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204712.169406:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204712.169413:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169418:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204712.169419:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169421:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169422:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204712.169426:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169431:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204712.169435:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169442:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204712.169444:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169445:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204712.169446:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169448:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204712.169449:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204712.169452:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204712.169455:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204712.169457:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204712.169462:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204712.169468:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205889920:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204712.169472:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169473:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204712.169476:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205890048/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169481:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204712.169482:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204712.169483:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204712.169487:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204712.169488:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204712.169489:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204712.169491:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204712.169493:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204712.169495:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204712.169496:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204712.169498:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:2.0:1788204712.169500:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204712.169516:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169518:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204712.169521:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205889792/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169525:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204712.169526:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204712.169528:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204712.169530:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204712.169531:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204712.169532:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204712.169534:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204712.169535:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204712.169537:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204712.169539:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204712.169540:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:2.0:1788204712.169542:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204712.169543:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169545:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204712.169548:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205889920/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204712.169553:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204712.169554:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204712.169555:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:2.0:1788204712.169558:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204712.169559:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204712.169560:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204712.169562:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204712.169563:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204712.169565:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204712.169566:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204712.169567:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:2.0:1788204712.169570:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204712.169571:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169572:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204712.169581:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204712.169582:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.169584:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204712.169585:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204712.360461:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204712.360467:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204712.360471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204712.360472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204712.360474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204712.360475:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204712.360477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204712.360478:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204712.360542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204712.360546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204712.360549:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204713.128446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204713.128453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204713.128456:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204713.128460:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204713.128463:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204713.128467:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204713.128470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204713.128472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204713.128475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204713.192439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204713.192444:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204713.192446:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204713.192449:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204713.192453:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204713.192455:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204713.192457:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204713.192459:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204713.192460:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204713.192504:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204713.192507:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204713.192508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204713.192510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204713.192512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204713.192513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204713.192515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204713.192516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204713.192518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204713.384495:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204713.384500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204713.384504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204713.384505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204713.384507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204713.384508:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204713.384510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204713.384511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204713.384524:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204713.384526:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204713.384527:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204713.704454:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300840449, at 4300840960 00000100:00000010:1.0:1788204713.704466:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000040:0.0:1788204713.704569:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300840449, at 4300840960 00000100:00000010:0.0:1788204713.704574:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000001:1.0:1788204714.152458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204714.152464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204714.152465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204714.152489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204714.152492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204714.152494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204714.152496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204714.152497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204714.152499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204714.216488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204714.216495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204714.216498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204714.216503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204714.216507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204714.216510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204714.216513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204714.216516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204714.216519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204714.216527:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204714.216530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204714.216531:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204714.216534:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204714.216535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204714.216537:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204714.216539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204714.216540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204714.216541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204714.216595:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204714.216601:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204714.216603:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204714.216606:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204714.216610:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204714.216617:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6169 00000400:00000200:2.0:1788204714.216635:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204714.216644:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204714.216647:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204714.216650:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216653:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216656:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216658:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204714.216662:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204714.216664:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204714.216666:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204714.216669:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204714.216671:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204714.216674:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204714.216676:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216678:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216680:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216681:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204714.216684:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204714.216687:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216689:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204714.216691:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204714.216695:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204714.216697:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216699:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204714.216702:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204714.216707:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204714.216711:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204714.216715:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204714.216716:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216719:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204714.216722:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204714.216726:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204714.216733:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204714.216740:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204714.216749:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205890176/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204714.216764:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204714.216767:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204714.216772:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205890176/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204714.216804:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204714.216807:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204714.216810:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204714.216811:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204714.216813:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204714.216815:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204714.216817:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6169 00000400:00000200:2.0:1788204714.216821:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204714.216824:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204714.216826:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204714.216828:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216830:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216832:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216834:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204714.216836:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204714.216837:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204714.216839:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204714.216840:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204714.216842:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204714.216844:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204714.216847:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216849:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216850:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216852:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204714.216855:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204714.216857:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216860:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204714.216861:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204714.216866:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204714.216868:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216871:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204714.216875:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204714.216879:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216881:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204714.216885:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204714.216887:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204714.216890:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204714.216891:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216892:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204714.216893:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204714.216895:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204714.216899:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204714.216908:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204714.216913:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205890304/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204714.216920:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204714.216922:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204714.216926:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205890304/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204714.216933:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204714.216935:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204714.216938:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204714.216940:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204714.216941:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204714.216944:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204714.216947:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6169 00000400:00000200:2.0:1788204714.216951:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204714.216954:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204714.216956:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204714.216958:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216960:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216962:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216965:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204714.216966:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204714.216968:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204714.216969:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204714.216971:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204714.216973:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204714.216974:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204714.216977:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216978:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.216980:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216982:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204714.216984:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204714.216986:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204714.216988:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204714.216990:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204714.216992:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204714.216994:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.216996:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204714.216998:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204714.217000:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.217002:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204714.217005:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204714.217007:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204714.217010:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204714.217011:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.217013:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204714.217014:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204714.217016:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204714.217019:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204714.217022:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204714.217026:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205890432/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204714.217033:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204714.217035:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204714.217039:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205890432/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204714.217046:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204714.217048:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204714.217051:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204714.217052:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204714.217054:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204714.217056:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204714.217058:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6169 00000400:00000200:2.0:1788204714.217062:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204714.217064:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204714.217066:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204714.217069:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.217071:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.217073:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.217074:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204714.217076:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204714.217077:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204714.217079:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204714.217080:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204714.217082:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204714.217084:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204714.217087:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.217089:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204714.217090:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204714.217092:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204714.217094:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204714.217096:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204714.217099:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204714.217100:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204714.217103:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204714.217104:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.217106:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204714.217108:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204714.217110:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.217111:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204714.217115:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204714.217116:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204714.217118:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204714.217119:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.217121:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204714.217122:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204714.217123:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204714.217127:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204714.217129:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204714.217133:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205890560/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204714.217138:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204714.217140:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204714.217143:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205890560/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204714.217150:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204714.217154:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6179) 00000800:00000200:3.1:1788204714.217581:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204714.217584:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204714.217903:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204714.217919:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204714.218147:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204714.218149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204714.218340:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204714.218342:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00010000:00000001:1.0:1788204714.408474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204714.408479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204714.408483:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204714.408484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204714.408486:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204714.408487:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204714.408489:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204714.408490:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204714.408543:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204714.408547:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204714.408550:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204715.176424:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204715.176429:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204715.176431:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204715.176435:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204715.176437:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204715.176440:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204715.176441:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204715.176443:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204715.176445:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204715.240442:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204715.240461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204715.240463:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204715.240467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204715.240470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204715.240473:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204715.240475:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204715.240477:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204715.240480:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204715.240558:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204715.240560:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204715.240561:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204715.240563:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204715.240564:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204715.240566:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204715.240567:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204715.240568:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204715.240570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204715.432493:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204715.432498:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204715.432501:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204715.432502:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204715.432504:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204715.432505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204715.432507:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204715.432507:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204715.432523:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204715.432524:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204715.432526:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204716.200479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204716.200484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204716.200486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204716.200489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204716.200492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204716.200495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204716.200497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204716.200498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204716.200500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204716.264481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204716.264481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204716.264487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204716.264488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204716.264490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204716.264491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204716.264495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204716.264497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204716.264499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204716.264502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204716.264503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204716.264506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204716.264508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204716.264510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204716.264511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204716.264516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204716.264519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204716.264521:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204716.456465:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204716.456470:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204716.456474:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204716.456479:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204716.456481:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204716.456482:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204716.456484:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204716.456485:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204716.456538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204716.456540:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204716.456543:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204717.224485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204717.224491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204717.224492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204717.224496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204717.224499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204717.224502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204717.224504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204717.224505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204717.224508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204717.288482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204717.288482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204717.288488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204717.288489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204717.288491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204717.288492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204717.288497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204717.288498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204717.288503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204717.288503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204717.288508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204717.288508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204717.288511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204717.288514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204717.288515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204717.288518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204717.288520:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204717.288522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204717.480500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204717.480505:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204717.480510:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204717.480511:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204717.480513:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204717.480514:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204717.480516:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204717.480517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204717.480528:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204717.480530:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204717.480531:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204718.248477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204718.248484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204718.248486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204718.248491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204718.248495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204718.248498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204718.248500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204718.248502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204718.248503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204718.312469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204718.312469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204718.312475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204718.312476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204718.312479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204718.312479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204718.312485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204718.312485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204718.312490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204718.312491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204718.312495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204718.312496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204718.312500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204718.312503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204718.312504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204718.312507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204718.312509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204718.312512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204718.504477:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204718.504484:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204718.504490:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204718.504491:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204718.504494:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204718.504496:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204718.504499:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204718.504500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204718.504563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204718.504566:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204718.504569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204718.824466:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300845570, at 4300846080 00000100:00000040:0.0:1788204718.824472:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300845570, at 4300846080 00000100:00000010:1.0:1788204718.824476:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000010:0.0:1788204718.824480:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000001:1.0:1788204719.272484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204719.272492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204719.272494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204719.272500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204719.272503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204719.272506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204719.272508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204719.272510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204719.272512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204719.336477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204719.336479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204719.336484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204719.336484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204719.336487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204719.336487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204719.336494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204719.336494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204719.336499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204719.336500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204719.336504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204719.336505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204719.336509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204719.336512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204719.336517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204719.336518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204719.336521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204719.336524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.336700:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204719.336705:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204719.336709:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204719.336727:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.336746:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204719.336749:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.336754:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204719.336757:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204719.336760:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204719.336766:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205890688:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204719.336772:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204719.336776:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204719.336780:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.336783:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204719.336789:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204719.336794:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204719.336800:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204719.336806:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204719.336812:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204719.336820:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205890688, portal 25 00000100:00000001:2.0:1788204719.336824:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204719.336827:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:2.0:1788204719.336837:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.336858:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204719.336867:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204719.336874:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204719.336881:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205890688, offset 0 00000400:00000200:2.0:1788204719.336895:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204719.336908:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.336919:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204719.336929:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204719.336944:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204719.336951:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204719.336958:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204719.336966:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58814:58814:256:4294967295] 192.168.201.151@tcp LPNI seq info [58794:58794:8:4294967295] 00000400:00000200:2.0:1788204719.336976:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.336993:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204719.337005:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337013:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204719.337024:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204719.337031:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204719.337040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204719.337052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204719.337063:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337067:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204719.337075:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204719.337085:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204719.337092:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204719.337096:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204719.337104:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204719.337112:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337131:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.337140:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337157:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337161:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204719.337169:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204719.337177:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204719.337182:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300850689 00000100:00000010:2.0:1788204719.337191:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1d20. 00000100:00000001:2.0:1788204719.337199:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204719.337209:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300850689 00000100:00000010:2.0:1788204719.337214:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1f60. 00000100:00000001:2.0:1788204719.337220:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204719.337235:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204719.337241:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337247:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204719.337256:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204719/real 1788204719] req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204719.337274:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204719.337277:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204719.337281:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204719.337287:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204719.337291:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204719.337296:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204719.337302:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204719.337307:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337312:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204719.337314:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337318:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337320:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204719.337325:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337335:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204719.337340:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337345:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204719.337347:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337348:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337349:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337351:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204719.337353:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204719.337356:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204719.337359:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204719.337362:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204719.337403:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204719.337412:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205890688:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204719.337416:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337418:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.337420:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205890688/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337425:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204719.337426:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204719.337428:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204719.337430:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204719.337431:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204719.337432:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204719.337434:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204719.337436:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204719.337438:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204719.337439:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204719.337440:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:2.0:1788204719.337442:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204719.337443:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337445:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337450:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204719.337451:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337452:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204719.337453:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204719.337460:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204719.337463:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337474:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204719.337475:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337477:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204719.337479:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204719.337481:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204719.337484:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205890816:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204719.337487:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204719.337489:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204719.337490:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337492:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204719.337495:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204719.337497:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204719.337501:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204719.337503:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204719.337506:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204719.337509:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205890816, portal 10 00000100:00000001:2.0:1788204719.337511:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204719.337512:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:2.0:1788204719.337517:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337521:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204719.337522:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204719.337524:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204719.337525:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205890816, offset 0 00000400:00000200:2.0:1788204719.337528:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337531:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337533:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204719.337535:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204719.337537:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204719.337538:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204719.337540:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337542:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58815:58815:256:4294967295] 192.168.201.151@tcp LPNI seq info [58795:58795:8:4294967295] 00000400:00000200:2.0:1788204719.337545:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337549:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204719.337552:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337554:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204719.337556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204719.337558:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204719.337561:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204719.337563:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204719.337565:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337566:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204719.337568:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204719.337570:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204719.337572:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204719.337573:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204719.337575:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204719.337577:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337581:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.337583:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337587:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337588:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204719.337590:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204719.337592:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204719.337594:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204719.337595:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300846747 00000100:00000001:2.0:1788204719.337597:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.337606:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300846747 00000100:00000001:2.0:1788204719.337607:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.337609:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204719.337610:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337611:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204719.337614:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204719/real 1788204719] req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204719.337619:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204719.337619:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204719.337620:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204719.337622:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204719.337623:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204719.337624:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204719.337626:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204719.337630:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337634:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204719.337634:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337635:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337637:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204719.337640:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337646:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204719.337650:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337656:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204719.337658:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337660:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337661:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337663:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204719.337665:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204719.337668:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204719.337671:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204719.337673:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204719.337677:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204719.337683:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205890816:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204719.337687:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204719.337690:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337695:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204719.337697:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337699:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204719.337700:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204719.337702:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204719.337705:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205890944:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204719.337708:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204719.337710:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204719.337711:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337713:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204719.337716:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c800. 02000000:00000001:2.0:1788204719.337718:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204719.337721:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204719.337724:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204719.337727:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204719.337729:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205890944, portal 4 00000100:00000001:2.0:1788204719.337730:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204719.337731:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:2.0:1788204719.337735:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337739:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204719.337741:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204719.337742:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204719.337743:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205890944, offset 0 00000400:00000200:2.0:1788204719.337746:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337748:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337750:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204719.337752:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204719.337754:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204719.337755:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204719.337757:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337758:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58816:58816:256:4294967295] 192.168.201.151@tcp LPNI seq info [58796:58796:8:4294967295] 00000400:00000200:2.0:1788204719.337761:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337765:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204719.337767:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337769:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204719.337771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204719.337773:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204719.337776:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204719.337777:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204719.337779:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337780:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204719.337782:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204719.337784:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204719.337786:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204719.337787:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204719.337789:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204719.337791:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337795:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.337797:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337801:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337802:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204719.337804:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204719.337805:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204719.337807:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204719.337808:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300846633 00000100:00000001:2.0:1788204719.337810:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.337811:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300846633 00000100:00000001:2.0:1788204719.337812:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.337813:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204719.337814:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.337815:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204719.337817:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204719/real 1788204719] req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204719.337822:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204719.337823:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204719.337824:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204719.337825:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204719.337826:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204719.337827:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204719.337828:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204719.337833:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337836:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204719.337837:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337838:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337839:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204719.337841:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337844:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204719.337847:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337854:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204719.337856:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337858:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.337859:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.337861:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204719.337863:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204719.337866:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204719.337869:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204719.337871:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204719.337875:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204719.337882:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205890944:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204719.337886:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204719.337889:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337894:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204719.337895:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337897:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204719.337899:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204719.337900:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204719.337903:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205891072:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204719.337906:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204719.337907:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204719.337909:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204719.337910:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204719.337913:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204719.337915:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204719.337918:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204719.337920:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204719.337922:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204719.337925:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205891072, portal 4 00000100:00000001:2.0:1788204719.337927:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204719.337928:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:2.0:1788204719.337933:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.337939:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204719.337941:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204719.337943:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204719.337945:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205891072, offset 0 00000400:00000200:2.0:1788204719.337949:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337952:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337954:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204719.337956:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204719.337958:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204719.337959:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204719.337961:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204719.337963:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58817:58817:256:4294967295] 192.168.201.151@tcp LPNI seq info [58797:58797:8:4294967295] 00000400:00000200:2.0:1788204719.337965:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204719.337969:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204719.337971:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337973:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204719.337976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204719.337977:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204719.337980:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204719.337981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204719.337994:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204719.337996:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204719.337997:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204719.337999:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204719.338002:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204719.338003:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204719.338005:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204719.338007:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338012:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.338014:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338017:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.338018:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204719.338020:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204719.338022:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204719.338024:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204719.338025:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300846639 00000100:00000001:2.0:1788204719.338026:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.338028:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300846639 00000100:00000001:2.0:1788204719.338029:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300850689 00000100:00000001:2.0:1788204719.338030:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204719.338031:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.338033:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204719.338034:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204719/real 1788204719] req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204719.338038:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204719.338039:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204719.338040:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204719.338041:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204719.338042:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204719.338043:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204719.338044:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204719.338050:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338061:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204719.338062:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338064:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.338065:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204719.338069:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338076:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204719.338079:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338086:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204719.338088:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338089:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204719.338090:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338091:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204719.338093:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204719.338094:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204719.338096:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204719.338097:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204719.338099:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204719.338104:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205891072:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204719.338106:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338107:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.338109:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205890816/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338112:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204719.338113:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204719.338115:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204719.338116:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204719.338117:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204719.338118:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204719.338119:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204719.338121:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204719.338122:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204719.338123:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204719.338124:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:2.0:1788204719.338126:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204719.338127:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338128:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.338129:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205890944/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338133:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204719.338133:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204719.338134:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c800. 02000000:00000001:2.0:1788204719.338135:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204719.338136:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204719.338137:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204719.338138:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204719.338138:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204719.338139:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204719.338140:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204719.338141:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:2.0:1788204719.338142:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204719.338143:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338144:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204719.338145:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205891072/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204719.338149:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204719.338149:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204719.338150:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204719.338151:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204719.338152:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204719.338152:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204719.338154:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204719.338155:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204719.338156:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204719.338156:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204719.338157:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204719.338158:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204719.338159:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338160:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.338162:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204719.338163:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.338164:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204719.338165:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528512:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204719.528519:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528524:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204719.528527:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204719.528529:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204719.528533:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528535:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204719.528537:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528539:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528541:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528545:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204719.528546:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204719.528549:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204719.528550:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204719.528566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204719.528568:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204719.528571:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204720.296475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204720.296481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204720.296483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204720.296488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204720.296491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204720.296494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204720.296496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204720.296499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204720.296501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204720.360480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204720.360480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204720.360488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204720.360488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204720.360492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204720.360496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204720.360498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204720.360501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204720.360505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204720.360507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204720.360509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204720.360511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204720.360514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204720.360518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204720.360521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204720.360522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204720.360525:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204720.360529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204720.552503:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204720.552510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204720.552514:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204720.552515:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204720.552517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204720.552518:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204720.552521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204720.552522:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204720.552563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204720.552567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204720.552570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204721.320487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204721.320496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204721.320498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204721.320503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204721.320505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204721.320509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204721.320511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204721.320513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204721.320515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204721.384495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204721.384496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204721.384506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204721.384509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204721.384510:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204721.384512:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204721.384517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204721.384519:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204721.384525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204721.384527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204721.384529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204721.384534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204721.384534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204721.384540:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204721.384544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204721.384545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204721.384550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204721.384555:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576540:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204721.576549:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204721.576552:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204721.576555:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576559:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204721.576561:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576562:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576564:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576567:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204721.576568:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576570:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204721.576571:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204721.576572:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204721.576573:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576574:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204721.576575:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576576:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576577:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576579:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204721.576580:0:48401:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204721.576581:0:48401:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204721.576582:0:48401:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576582:0:48401:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204721.576583:0:48401:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576584:0:48401:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204721.576585:0:48401:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204721.576604:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204721.576608:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204721.576610:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204722.344449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204722.344457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204722.344459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204722.344464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204722.344467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204722.344470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204722.344472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204722.344475:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204722.344477:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204722.408478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204722.408478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204722.408488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204722.408488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204722.408491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204722.408492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204722.408498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204722.408499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204722.408503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204722.408504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204722.408507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204722.408508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204722.408511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204722.408516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204722.408520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204722.408521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204722.408524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204722.408528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204722.600511:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204722.600520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204722.600527:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204722.600528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204722.600531:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204722.600532:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204722.600536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204722.600537:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204722.600602:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204722.600607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204722.600611:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204723.368485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204723.368496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204723.368499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204723.368504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204723.368508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204723.368511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204723.368513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204723.368515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204723.368518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204723.432498:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204723.432498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204723.432509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204723.432509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204723.432513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204723.432514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204723.432522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204723.432522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204723.432527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204723.432528:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204723.432535:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204723.432535:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204723.432542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204723.432545:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204723.432549:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204723.432549:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204723.432553:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204723.432558:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204723.624542:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204723.624553:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204723.624560:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204723.624562:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204723.624566:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204723.624567:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204723.624571:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204723.624572:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204723.624600:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204723.624604:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204723.624607:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204723.944457:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300850689, at 4300851200 00000100:00000040:0.0:1788204723.944462:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300850689, at 4300851200 00000100:00000010:0.0:1788204723.944477:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000010:1.0:1788204723.944481:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000001:1.0:1788204724.392441:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204724.392449:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204724.392451:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204724.392457:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204724.392462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204724.392466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204724.392469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204724.392477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204724.392480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204724.456495:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204724.456495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204724.456504:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204724.456505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204724.456506:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204724.456508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204724.456515:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204724.456516:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204724.456517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204724.456520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204724.456522:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204724.456523:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204724.456524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204724.456528:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204724.456529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204724.456530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204724.456533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204724.456535:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204724.456537:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204724.456538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204724.456540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204724.456544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204724.456548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204724.456549:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6179 00000400:00000200:2.0:1788204724.456562:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204724.456570:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204724.456585:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204724.456589:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456593:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.456596:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456598:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204724.456602:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204724.456604:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204724.456607:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204724.456610:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204724.456612:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204724.456615:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204724.456617:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456620:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.456622:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456624:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204724.456627:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204724.456630:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456633:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204724.456637:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204724.456642:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204724.456645:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.456648:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204724.456651:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204724.456661:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204724.456666:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204724.456670:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204724.456672:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.456676:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204724.456679:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204724.456684:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204724.456693:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204724.456702:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204724.456719:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205891200/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204724.456727:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204724.456731:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204724.456735:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205891200/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204724.456753:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204724.456755:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204724.456757:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204724.456758:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204724.456759:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204724.456761:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204724.456764:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6179 00000400:00000200:2.0:1788204724.456768:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204724.456770:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204724.456772:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204724.456774:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456776:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.456778:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456781:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204724.456783:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204724.456785:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204724.456788:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204724.456792:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204724.456797:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204724.456800:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204724.456809:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456812:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.456814:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456816:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204724.456821:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204724.456828:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456832:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204724.456837:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204724.456845:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204724.456849:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.456852:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204724.456860:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204724.456865:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.456869:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204724.456884:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204724.456889:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204724.456895:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204724.456899:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.456902:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204724.456904:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204724.456906:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204724.456911:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204724.456915:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204724.456923:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205891328/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204724.456933:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204724.456937:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204724.456944:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205891328/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204724.456956:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204724.456960:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204724.456962:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204724.456963:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204724.456964:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204724.456968:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204724.456972:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6179 00000400:00000200:2.0:1788204724.456977:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204724.456980:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204724.456982:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204724.456985:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456989:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.456992:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.456994:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204724.456996:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204724.456999:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204724.457007:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204724.457010:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204724.457014:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204724.457019:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204724.457025:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457027:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.457032:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457036:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204724.457039:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204724.457042:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457047:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204724.457050:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204724.457054:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204724.457059:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457063:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204724.457065:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204724.457068:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457071:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204724.457074:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204724.457077:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204724.457081:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204724.457084:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457086:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204724.457089:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788204724.457090:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788204724.457091:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204724.457095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 02000000:00000010:2.0:1788204724.457097:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788204724.457100:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204724.457105:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205891456/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204724.457112:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204724.457115:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204724.457121:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205891456/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204724.457132:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204724.457135:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204724.457140:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204724.457142:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204724.457144:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204724.457149:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204724.457154:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6179 00000400:00000200:2.0:1788204724.457158:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204724.457163:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204724.457164:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204724.457169:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457172:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.457175:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457180:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204724.457185:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204724.457187:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204724.457191:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204724.457195:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204724.457198:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204724.457202:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204724.457210:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457214:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204724.457219:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457222:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204724.457227:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204724.457229:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204724.457231:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204724.457232:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204724.457235:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204724.457238:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457240:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204724.457242:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204724.457246:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457248:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204724.457254:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204724.457257:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204724.457269:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204724.457271:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.457275:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204724.457277:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204724.457278:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204724.457281:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788204724.457283:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204724.457301:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205891584/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204724.457307:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204724.457308:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204724.457311:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205891584/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204724.457320:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204724.457325:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6189) 00000800:00000200:3.1:1788204724.457564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204724.457567:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204724.457827:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204724.457828:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204724.458011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204724.458013:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00010000:00000001:1.0:1788204724.648491:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204724.648500:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204724.648515:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204724.648517:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204724.648522:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204724.648523:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204724.648526:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204724.648527:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204724.648580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204724.648590:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204724.648593:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204725.416516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204725.416524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204725.416526:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204725.416531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204725.416534:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204725.416538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204725.416540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204725.416543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204725.416545:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204725.480437:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204725.480437:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204725.480450:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204725.480454:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204725.480454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204725.480457:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204725.480462:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204725.480463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204725.480470:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204725.480470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204725.480477:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204725.480477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204725.480483:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204725.480486:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204725.480490:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204725.480491:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204725.480494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204725.480500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204725.672521:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204725.672528:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204725.672532:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204725.672533:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204725.672536:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204725.672536:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204725.672539:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204725.672540:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204725.672555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204725.672557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204725.672559:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204725.992480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204725.992490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204725.992493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204725.992501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204725.992505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204725.992510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204725.992513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204725.992517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204725.992520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204726.440528:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204726.440539:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204726.440543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204726.440551:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204726.440555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204726.440560:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204726.440563:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204726.440566:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204726.440569:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.504482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204726.504482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204726.504494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204726.504494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204726.504497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204726.504498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204726.504505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204726.504505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204726.504512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204726.504512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204726.504516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204726.504519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204726.504522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204726.504523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204726.504525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204726.504528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204726.504531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204726.504535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204726.696454:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204726.696463:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204726.696471:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204726.696472:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204726.696476:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204726.696477:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204726.696480:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204726.696481:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204726.696534:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204726.696543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204726.696546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204727.016544:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204727.016555:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204727.016558:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204727.016564:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204727.016568:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204727.016574:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204727.016577:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204727.016581:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204727.016583:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204727.464525:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204727.464536:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204727.464539:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204727.464555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204727.464560:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204727.464566:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204727.464569:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204727.464572:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204727.464577:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204727.528470:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204727.528477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204727.528479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204727.528483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204727.528487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204727.528492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204727.528495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204727.528498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204727.528502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204727.528568:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204727.528574:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204727.528576:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204727.528580:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204727.528584:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204727.528588:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204727.528591:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204727.528594:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204727.528597:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204727.720614:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204727.720622:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204727.720627:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204727.720628:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204727.720631:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204727.720632:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204727.720634:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204727.720635:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204727.720655:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204727.720657:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204727.720660:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204728.040498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204728.040506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204728.040508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204728.040513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204728.040516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204728.040521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204728.040523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204728.040526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204728.040528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204728.488489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204728.488497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204728.488499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204728.488509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204728.488513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204728.488516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204728.488518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204728.488521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204728.488523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204728.552477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204728.552485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204728.552487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204728.552493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204728.552496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204728.552496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204728.552502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204728.552507:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204728.552507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204728.552511:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204728.552513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204728.552520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204728.552521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204728.552529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204728.552534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204728.552538:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204728.552540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204728.552543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204728.744500:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204728.744510:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204728.744517:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204728.744520:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204728.744525:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204728.744535:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204728.744538:0:48401:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204728.744539:0:48401:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204728.744604:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204728.744609:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204728.744614:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204729.064469:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300855809, at 4300856320 00000100:00000040:1.0:1788204729.064469:0:48401:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300855809, at 4300856320 00000100:00000001:2.0:1788204729.064475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204729.064481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204729.064484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204729.064486:0:48401:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000010:0.0:1788204729.064488:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00100000:2.0:1788204729.064488:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204729.064493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204729.064497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204729.064499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204729.064501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204729.064504:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204729.512513:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204729.512525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204729.512527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204729.512534:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204729.512538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204729.512542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204729.512545:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204729.512555:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204729.512558:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204729.576474:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204729.576476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204729.576476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204729.576482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204729.576483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204729.576484:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204729.576485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204729.576487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576488:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204729.576495:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204729.576496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204729.576500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204729.576502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204729.576502:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204729.576506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204729.576510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204729.576511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204729.576511:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6184 00000100:00000001:1.0:1788204729.576515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204729.576520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204729.576524:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204729.576525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204729.576526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204729.576530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204729.576531:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788204729.576537:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204729.576537:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204729.576553:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576556:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576559:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576560:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204729.576564:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204729.576566:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204729.576568:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204729.576573:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204729.576575:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204729.576577:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204729.576581:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576583:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576588:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576590:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204729.576593:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204729.576596:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576599:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204729.576602:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204729.576607:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204729.576610:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576613:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204729.576616:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204729.576626:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204729.576631:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204729.576635:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204729.576637:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576641:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204729.576643:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204729.576648:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204729.576658:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204729.576666:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204729.576679:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205891712/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204729.576689:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204729.576695:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204729.576699:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205891712/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204729.576726:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204729.576730:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204729.576733:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204729.576735:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204729.576736:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204729.576739:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204729.576742:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6184 00000400:00000200:2.0:1788204729.576745:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204729.576749:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204729.576751:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204729.576755:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576757:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576760:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576763:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204729.576765:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204729.576766:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204729.576769:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204729.576773:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204729.576775:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204729.576779:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204729.576784:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576786:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576790:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576791:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204729.576794:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204729.576798:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576801:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204729.576804:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204729.576810:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204729.576814:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576816:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204729.576821:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204729.576826:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576830:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204729.576836:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204729.576841:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204729.576846:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204729.576848:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576851:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204729.576855:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204729.576858:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204729.576865:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204729.576870:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204729.576877:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205891840/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204729.576884:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204729.576887:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204729.576890:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205891840/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204729.576899:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204729.576901:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204729.576904:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204729.576907:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204729.576909:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204729.576914:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204729.576917:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6184 00000400:00000200:2.0:1788204729.576923:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204729.576927:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204729.576931:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204729.576936:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576940:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576942:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576945:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204729.576947:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204729.576949:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204729.576950:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204729.576953:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204729.576956:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204729.576959:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204729.576963:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576964:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.576968:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576970:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204729.576972:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204729.576975:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204729.576977:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204729.576980:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204729.576984:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204729.576987:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.576991:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204729.576994:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204729.576998:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.577000:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204729.577007:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204729.577013:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204729.577018:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204729.577022:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.577025:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204729.577028:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204729.577030:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204729.577035:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204729.577040:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204729.577048:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205891968/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204729.577054:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204729.577055:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204729.577058:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205891968/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204729.577065:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204729.577069:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204729.577073:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204729.577076:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204729.577078:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204729.577081:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204729.577085:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6184 00000400:00000200:2.0:1788204729.577090:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204729.577093:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204729.577095:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204729.577100:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.577102:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204729.577104:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.577108:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204729.577110:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204729.577112:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204729.577114:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204729.577117:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204729.577119:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204729.577123:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204729.577127:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204729.577128:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204729.577130:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204729.577134:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1788204729.577135:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204729.577140:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204729.577145:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204729.577152:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204729.577157:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204729.577159:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204729.577162:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204729.577166:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.577170:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204729.577174:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204729.577178:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.577180:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204729.577187:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204729.577191:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204729.577195:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204729.577197:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204729.577214:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204729.577216:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204729.577218:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204729.577223:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204729.577227:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204729.577234:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205892096/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204729.577256:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204729.577258:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204729.577263:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205892096/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204729.577269:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204729.577273:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6194) 00000800:00000200:3.1:1788204729.577641:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204729.577643:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204729.577908:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204729.577910:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204729.578159:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204729.578161:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00010000:00000001:1.0:1788204729.768531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204729.768538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204729.768544:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204729.768547:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204729.768549:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768552:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204729.768553:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768555:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768557:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204729.768560:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204729.768562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204729.768563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204729.768577:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204729.768579:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204729.768581:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204730.088505:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204730.088514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204730.088515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204730.088522:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204730.088527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204730.088531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204730.088534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204730.088538:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204730.088541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204730.536450:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204730.536462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204730.536465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204730.536471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204730.536476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204730.536480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204730.536483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204730.536486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204730.536490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204730.600438:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204730.600445:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204730.600447:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204730.600451:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204730.600455:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204730.600458:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204730.600460:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204730.600462:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204730.600465:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204730.601406:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204730.601410:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204730.601411:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204730.601414:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204730.601416:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204730.601418:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204730.601420:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204730.601423:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204730.601425:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204730.792473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204730.792478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204730.792482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204730.792483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204730.792486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204730.792487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204730.792490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204730.792491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204730.792556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204730.792560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204730.792563:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204731.112482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204731.112490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204731.112496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.112501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204731.112505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204731.112508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204731.112511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204731.112513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204731.112515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204731.560471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204731.560479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204731.560481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204731.560486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204731.560489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204731.560493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204731.560496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204731.560498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204731.560501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204731.624469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204731.624476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204731.624479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204731.624484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204731.624488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204731.624491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204731.624494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204731.624496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204731.624499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204731.624505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204731.624509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204731.624510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204731.624513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204731.624516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204731.624519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204731.624521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204731.624523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204731.624525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204731.816538:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204731.816540:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204731.816543:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816549:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204731.816551:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816552:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816554:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204731.816557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204731.816561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204731.816562:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204731.816563:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816564:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204731.816565:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816566:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816567:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816569:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204731.816570:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204731.816571:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204731.816571:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816572:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204731.816573:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816574:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204731.816575:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204731.816589:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204731.816591:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204731.816593:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204732.136487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204732.136497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204732.136500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204732.136507:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204732.136511:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204732.136514:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204732.136516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204732.136518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204732.136520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204732.584437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204732.584445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204732.584447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204732.584452:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204732.584455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204732.584459:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204732.584461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204732.584463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204732.584466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204732.648445:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204732.648445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204732.648455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204732.648455:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204732.648459:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204732.648460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204732.648468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204732.648468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204732.648475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204732.648475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204732.648482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204732.648482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204732.648487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204732.648490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204732.648493:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204732.648494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204732.648497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204732.648500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204732.840466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204732.840476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204732.840483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204732.840485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204732.840490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204732.840491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204732.840495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204732.840497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204732.840549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204732.840553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204732.840556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204733.160506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204733.160515:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204733.160517:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204733.160522:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204733.160526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204733.160530:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204733.160532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204733.160534:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204733.160536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204733.608481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204733.608488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204733.608491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204733.608496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204733.608499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204733.608504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204733.608506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204733.608508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204733.608511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204733.672481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204733.672493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204733.672496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204733.672504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204733.672509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204733.672511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204733.672514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204733.672515:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204733.672519:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204733.672519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788204733.672524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204733.672525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204733.672529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204733.672532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204733.672533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204733.672537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204733.672540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204733.672542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204733.864502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204733.864510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204733.864515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204733.864516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204733.864518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204733.864519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204733.864521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204733.864522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204733.864536:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204733.864538:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204733.864540:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204734.184437:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300860929, at 4300861440 00000100:00000040:1.0:1788204734.184437:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300860929, at 4300861440 00000100:00000001:2.0:1788204734.184438:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:1.0:1788204734.184451:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000001:2.0:1788204734.184452:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204734.184457:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204734.184459:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00100000:2.0:1788204734.184462:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204734.184466:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204734.184470:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204734.184472:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204734.184474:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204734.184477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204734.632455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204734.632466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204734.632468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204734.632475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204734.632479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204734.632484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204734.632486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204734.632488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204734.632491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204734.696490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204734.696490:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204734.696491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204734.696499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204734.696500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204734.696501:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204734.696503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204734.696504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696506:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204734.696511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204734.696511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204734.696511:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204734.696518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204734.696518:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204734.696518:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204734.696522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204734.696523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204734.696526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204734.696527:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6189 00000100:00000001:0.0:1788204734.696529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204734.696533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204734.696534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204734.696537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204734.696540:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204734.696541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204734.696555:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204734.696559:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204734.696563:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696566:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696569:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696571:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204734.696575:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204734.696577:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204734.696581:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204734.696584:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204734.696586:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204734.696588:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204734.696590:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696591:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696592:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696593:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204734.696595:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204734.696598:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696599:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204734.696602:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204734.696605:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204734.696607:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696609:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204734.696611:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204734.696621:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204734.696624:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204734.696628:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204734.696629:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696632:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204734.696634:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204734.696638:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204734.696646:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204734.696653:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204734.696662:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205892224/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204734.696670:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204734.696673:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204734.696676:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205892224/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204734.696699:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204734.696701:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204734.696704:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204734.696705:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204734.696706:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204734.696708:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204734.696710:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6189 00000400:00000200:2.0:1788204734.696713:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204734.696715:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204734.696717:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204734.696719:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696721:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696723:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696725:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204734.696727:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204734.696727:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204734.696729:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204734.696735:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204734.696738:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204734.696741:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204734.696745:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696747:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696751:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696756:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204734.696761:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204734.696767:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696774:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204734.696777:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204734.696785:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204734.696791:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696795:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204734.696803:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204734.696808:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696810:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204734.696814:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204734.696818:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204734.696820:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204734.696823:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696824:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204734.696825:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204734.696827:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204734.696831:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204734.696843:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204734.696847:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205892352/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204734.696855:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204734.696858:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204734.696863:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205892352/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204734.696877:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204734.696882:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204734.696893:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204734.696896:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204734.696898:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204734.696901:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204734.696905:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6189 00000400:00000200:2.0:1788204734.696911:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204734.696915:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204734.696917:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204734.696923:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696927:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696930:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696935:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204734.696937:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204734.696940:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204734.696942:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204734.696945:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204734.696948:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204734.696953:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204734.696958:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696962:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.696964:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696968:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204734.696974:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204734.696979:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204734.696982:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204734.696986:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204734.696990:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204734.696995:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.696999:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204734.697002:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204734.697006:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.697011:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204734.697016:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204734.697020:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204734.697026:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204734.697028:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.697031:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204734.697033:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204734.697037:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204734.697043:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204734.697048:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204734.697056:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205892480/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204734.697070:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204734.697073:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204734.697080:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205892480/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204734.697096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204734.697097:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204734.697101:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204734.697101:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00080000:2.0:1788204734.697107:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204734.697109:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204734.697124:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204734.697130:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204734.697134:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6189 00000400:00000200:2.0:1788204734.697138:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204734.697140:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204734.697142:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204734.697147:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.697149:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.697152:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.697155:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204734.697159:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204734.697160:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204734.697162:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204734.697163:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204734.697165:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204734.697168:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204734.697172:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.697174:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204734.697177:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204734.697180:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204734.697182:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204734.697186:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204734.697190:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204734.697197:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204734.697200:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204734.697204:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.697208:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204734.697211:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204734.697214:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.697219:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204734.697226:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204734.697228:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204734.697232:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204734.697234:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204734.697236:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204734.697238:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204734.697240:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204734.697245:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204734.697249:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204734.697254:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205892608/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204734.697266:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204734.697269:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204734.697274:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205892608/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204734.697291:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204734.697298:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6199) 00000800:00000200:3.1:1788204734.697564:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204734.697578:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204734.697764:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204734.697766:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204734.697977:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204734.697981:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00010000:00000001:1.0:1788204734.888488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204734.888496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204734.888501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204734.888502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204734.888505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204734.888505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204734.888507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204734.888508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204734.888561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204734.888566:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204734.888570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204735.208482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204735.208491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204735.208492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204735.208498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204735.208501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204735.208505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204735.208507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204735.208509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204735.208512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204735.656459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204735.656470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204735.656473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204735.656481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204735.656486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204735.656492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204735.656496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204735.656499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204735.656504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204735.720492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204735.720492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204735.720502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204735.720503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204735.720505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204735.720509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204735.720513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204735.720517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204735.720519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204735.720525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204735.720526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204735.720531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204735.720534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204735.720535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204735.720543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204735.720543:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204735.720548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204735.720552:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204735.912533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204735.912542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204735.912547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204735.912548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204735.912550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204735.912551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204735.912554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204735.912555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204735.912572:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204735.912574:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204735.912577:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204736.680436:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204736.680445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204736.680447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204736.680452:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204736.680456:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204736.680460:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204736.680462:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204736.680464:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204736.680467:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204736.744527:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204736.744527:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204736.744537:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204736.744537:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204736.744539:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204736.744541:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204736.744547:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204736.744549:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204736.744554:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204736.744555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204736.744559:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204736.744560:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204736.744564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204736.744567:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204736.744570:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204736.744573:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204736.744577:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204736.744580:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204736.936495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204736.936505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204736.936514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204736.936516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204736.936521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204736.936522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204736.936527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204736.936529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204736.936588:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204736.936592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204736.936597:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204737.704517:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204737.704529:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204737.704532:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204737.704540:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204737.704545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204737.704551:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204737.704554:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204737.704557:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204737.704560:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204737.768508:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204737.768508:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204737.768517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204737.768517:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204737.768520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204737.768520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204737.768527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204737.768527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204737.768531:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204737.768531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204737.768537:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204737.768537:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204737.768541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204737.768544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204737.768548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204737.768548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204737.768549:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204737.768551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204737.960580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204737.960588:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204737.960594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204737.960595:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204737.960598:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204737.960599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204737.960602:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204737.960603:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204737.960621:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204737.960624:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204737.960626:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204738.728483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204738.728492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204738.728494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204738.728500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204738.728504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204738.728508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204738.728511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204738.728514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204738.728517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204738.792503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204738.792508:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204738.792512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204738.792516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204738.792516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204738.792521:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204738.792526:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204738.792527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204738.792535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204738.792535:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204738.792546:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204738.792546:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204738.792571:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204738.792574:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204738.792578:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204738.792579:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204738.792583:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204738.792586:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204738.984516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204738.984524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204738.984532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204738.984535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204738.984539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204738.984541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204738.984546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204738.984548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204738.984613:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204738.984626:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204738.984633:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204739.304469:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300866049, at 4300866560 00000100:00000040:1.0:1788204739.304469:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300866049, at 4300866560 00000100:00000010:1.0:1788204739.304488:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000010:0.0:1788204739.304491:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000001:1.0:1788204739.752496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204739.752508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204739.752511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204739.752518:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204739.752523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204739.752528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204739.752532:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204739.752535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204739.752540:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204739.816483:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204739.816489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204739.816489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204739.816497:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204739.816498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204739.816498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204739.816500:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204739.816503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204739.816504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204739.816506:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204739.816512:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204739.816513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204739.816513:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204739.816520:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204739.816520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204739.816522:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6194 00000100:00000001:1.0:1788204739.816526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204739.816527:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204739.816530:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1788204739.816534:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204739.816535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204739.816540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204739.816540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204739.816542:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788204739.816546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204739.816547:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788204739.816551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.816552:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816558:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.816561:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816563:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204739.816572:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204739.816574:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204739.816577:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204739.816580:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204739.816582:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204739.816584:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204739.816586:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.816589:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816590:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204739.816592:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204739.816595:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816596:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204739.816599:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204739.816602:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204739.816604:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.816606:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204739.816608:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204739.816618:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204739.816621:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204739.816624:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204739.816626:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.816628:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204739.816630:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204739.816635:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204739.816648:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788204739.816657:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204739.816666:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205892736/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204739.816679:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204739.816683:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.816687:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205892736/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204739.816706:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204739.816709:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204739.816711:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204739.816713:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204739.816714:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204739.816716:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204739.816718:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6194 00000400:00000200:2.0:1788204739.816721:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204739.816723:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204739.816725:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204739.816727:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816728:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.816731:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816732:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204739.816735:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204739.816736:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204739.816737:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204739.816739:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204739.816741:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204739.816742:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204739.816744:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816746:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.816747:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816749:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204739.816754:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204739.816758:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816763:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204739.816764:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204739.816772:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204739.816779:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.816784:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204739.816792:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204739.816799:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.816804:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204739.816811:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204739.816818:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204739.816823:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204739.816825:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.816827:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204739.816830:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204739.816833:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204739.816840:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204739.816845:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204739.816854:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205892864/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204739.816861:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204739.816865:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.816870:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205892864/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204739.816881:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204739.816886:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204739.816894:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204739.816897:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204739.816901:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204739.816906:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204739.816911:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6194 00000400:00000200:2.0:1788204739.816919:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204739.816925:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204739.816930:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204739.816936:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816941:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.816948:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.816952:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204739.816958:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204739.816963:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204739.816967:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204739.816973:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204739.816980:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204739.816987:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204739.816994:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817000:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.817006:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817010:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204739.817014:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.1:1788204739.817049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:2.0:1788204739.817050:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.1:1788204739.817052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1788204739.817057:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204739.817063:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204739.817069:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204739.817074:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817079:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204739.817082:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204739.817086:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817088:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204739.817093:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204739.817097:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204739.817101:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204739.817103:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817108:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204739.817111:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204739.817114:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204739.817120:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204739.817126:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204739.817133:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205892992/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204739.817162:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204739.817167:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.817172:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205892992/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204739.817183:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204739.817186:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204739.817189:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204739.817190:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204739.817191:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204739.817193:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204739.817198:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6194 00000400:00000200:2.0:1788204739.817205:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204739.817207:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204739.817209:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204739.817216:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817217:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.817219:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817221:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204739.817223:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204739.817226:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204739.817228:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204739.817230:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204739.817232:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204739.817238:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204739.817243:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817247:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204739.817250:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817256:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204739.817259:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204739.817264:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204739.817266:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204739.817269:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204739.817271:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204739.817276:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817279:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204739.817282:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204739.817285:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817286:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204739.817292:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204739.817297:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204739.817301:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204739.817304:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204739.817307:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204739.817310:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204739.817313:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204739.817319:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204739.817324:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204739.817332:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205893120/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204739.817345:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204739.817348:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.817354:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205893120/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204739.817384:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204739.817389:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6204) 00000800:00000200:3.1:1788204739.817587:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204739.817589:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204739.817782:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204739.817784:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204739.817958:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204739.817960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00010000:00000001:1.0:1788204740.008534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204740.008541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204740.008550:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204740.008552:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204740.008555:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008558:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204740.008560:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008562:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008564:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204740.008567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204740.008571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204740.008572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204740.008587:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204740.008590:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204740.008592:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204740.776530:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204740.776541:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204740.776544:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204740.776552:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204740.776556:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204740.776562:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204740.776566:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204740.776569:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204740.776573:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204740.840481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204740.840489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204740.840491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204740.840498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204740.840499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204740.840502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204740.840506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204740.840508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204740.840509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204740.840512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788204740.840514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204740.840516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204740.840518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204740.840520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204740.840521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204740.840525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204740.840527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204740.840529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204741.032511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204741.032521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204741.032530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204741.032531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204741.032535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204741.032537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204741.032541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204741.032543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204741.032612:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204741.032617:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204741.032622:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204741.800511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204741.800524:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204741.800526:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204741.800533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204741.800537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204741.800541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204741.800543:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204741.800546:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204741.800549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204741.864479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204741.864482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204741.864489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204741.864490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204741.864494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204741.864494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204741.864502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204741.864503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204741.864508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204741.864508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204741.864514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204741.864514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204741.864520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204741.864523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204741.864527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204741.864527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204741.864531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204741.864535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204742.056560:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204742.056563:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204742.056567:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056575:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204742.056578:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056580:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056583:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204742.056590:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204742.056596:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204742.056597:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204742.056599:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056601:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204742.056603:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056605:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056607:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204742.056614:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204742.056615:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204742.056617:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056619:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204742.056620:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056622:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204742.056624:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204742.056647:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204742.056651:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204742.056655:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204742.824480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204742.824489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204742.824492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204742.824497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204742.824501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204742.824505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204742.824508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204742.824510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204742.824513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204742.888516:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204742.888525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204742.888528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204742.888533:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204742.888535:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204742.888541:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204742.888542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204742.888546:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204742.888549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204742.888552:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204742.888555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204742.888557:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204742.888561:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204742.888563:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204742.888568:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204742.888569:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204742.888574:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204742.888580:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204743.080480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204743.080486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204743.080490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204743.080491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204743.080494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204743.080495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204743.080498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204743.080499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204743.080563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204743.080566:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204743.080569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204743.848477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204743.848485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204743.848488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204743.848493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204743.848497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204743.848500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204743.848503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204743.848505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204743.848508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204743.912488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204743.912498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204743.912500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204743.912506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204743.912510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204743.912514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204743.912517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204743.912519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204743.912522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204743.912537:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204743.912543:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204743.912545:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204743.912549:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204743.912553:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204743.912555:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204743.912557:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204743.912559:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204743.912561:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204744.104522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204744.104530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204744.104536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204744.104537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204744.104540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204744.104541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204744.104544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204744.104545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204744.104565:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204744.104567:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204744.104570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204744.424524:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300871169, at 4300871680 00000100:00000040:1.0:1788204744.424524:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300871169, at 4300871680 00000100:00000010:0.0:1788204744.424546:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000010:1.0:1788204744.424549:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000001:1.0:1788204744.872468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204744.872474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204744.872477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204744.872481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204744.872484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204744.872488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204744.872490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204744.872493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204744.872496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204744.936439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204744.936443:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204744.936446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204744.936451:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204744.936451:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204744.936455:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204744.936457:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204744.936459:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204744.936462:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204744.936464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204744.936464:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204744.936466:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204744.936469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204744.936470:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204744.936472:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6199 00000100:00000001:0.0:1788204744.936473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204744.936474:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204744.936479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204744.936480:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204744.936486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1788204744.936486:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204744.936490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204744.936493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204744.936496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204744.936499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204744.936504:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204744.936509:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204744.936512:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936517:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936520:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936523:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204744.936527:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204744.936530:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204744.936534:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204744.936538:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204744.936540:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204744.936543:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204744.936547:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936550:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936552:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936554:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204744.936558:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204744.936561:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936564:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204744.936567:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204744.936571:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204744.936574:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936577:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204744.936580:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204744.936588:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204744.936591:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204744.936594:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204744.936596:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936598:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204744.936600:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204744.936604:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204744.936610:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204744.936615:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204744.936623:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205893248/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204744.936630:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204744.936634:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204744.936638:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205893248/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204744.936657:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204744.936659:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204744.936662:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204744.936663:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204744.936664:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204744.936666:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204744.936668:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6199 00000400:00000200:2.0:1788204744.936671:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204744.936673:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204744.936674:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204744.936676:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936678:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936679:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936681:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204744.936689:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204744.936690:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204744.936691:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204744.936693:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204744.936695:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204744.936699:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204744.936702:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936703:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936705:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936706:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204744.936708:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204744.936713:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936716:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204744.936719:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204744.936724:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204744.936727:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936728:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204744.936733:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204744.936738:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936742:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204744.936747:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204744.936751:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204744.936755:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204744.936757:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936759:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204744.936763:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204744.936766:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204744.936771:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204744.936777:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204744.936787:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205893376/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204744.936798:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204744.936802:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204744.936808:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205893376/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204744.936814:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204744.936816:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204744.936821:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204744.936824:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204744.936826:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204744.936831:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204744.936834:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6199 00000400:00000200:2.0:1788204744.936838:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204744.936841:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204744.936843:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204744.936846:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936850:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936854:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936857:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204744.936859:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204744.936861:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204744.936862:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204744.936865:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204744.936869:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204744.936874:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204744.936881:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936884:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.936889:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936893:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204744.936896:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204744.936900:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204744.936906:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204744.936907:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204744.936912:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204744.936916:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936917:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204744.936919:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204744.936920:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936921:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204744.936924:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204744.936927:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204744.936931:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204744.936932:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.936933:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204744.936950:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204744.936952:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204744.936955:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204744.936958:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204744.936964:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205893504/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204744.936975:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204744.936977:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204744.936982:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205893504/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204744.936994:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204744.936998:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204744.937002:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204744.937006:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204744.937009:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204744.937014:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204744.937018:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6199 00000400:00000200:2.0:1788204744.937022:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204744.937029:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204744.937032:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204744.937038:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.937040:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.937041:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.937043:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204744.937045:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204744.937045:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204744.937048:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204744.937051:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204744.937056:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204744.937058:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204744.937062:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.937064:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204744.937066:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204744.937068:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204744.937071:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204744.937077:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204744.937079:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204744.937083:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204744.937087:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1788204744.937089:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788204744.937090:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204744.937094:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1788204744.937095:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204744.937098:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204744.937103:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.937107:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204744.937114:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204744.937118:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204744.937122:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204744.937125:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.937128:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204744.937130:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204744.937133:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204744.937138:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204744.937141:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204744.937147:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205893632/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204744.937168:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204744.937171:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204744.937180:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205893632/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204744.937207:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204744.937216:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6209) 00000800:00000200:3.1:1788204744.937533:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204744.937536:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204744.937777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204744.937778:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204744.938000:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204744.938001:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00010000:00000001:1.0:1788204745.128491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204745.128499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204745.128506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204745.128507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204745.128510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204745.128512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204745.128515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204745.128517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204745.128615:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204745.128620:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204745.128623:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204745.896458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204745.896465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204745.896466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204745.896471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204745.896474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204745.896477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204745.896478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204745.896480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204745.896483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204745.960461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204745.960468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204745.960470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204745.960474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204745.960477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204745.960480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204745.960482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204745.960484:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204745.960486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204745.960497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204745.960502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204745.960503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204745.960506:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204745.960509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204745.960511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204745.960513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204745.960515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204745.960516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204746.152481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204746.152487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204746.152490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204746.152492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204746.152494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204746.152495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204746.152497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204746.152498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204746.152511:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204746.152513:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204746.152516:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204746.920479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204746.920484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204746.920486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204746.920491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204746.920494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204746.920498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204746.920501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204746.920504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204746.920508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204746.984451:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204746.984458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204746.984461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204746.984466:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204746.984471:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204746.984474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204746.984477:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204746.984478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204746.984480:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204746.984484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204746.984486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204746.984488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204746.984494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204746.984498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204746.984501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204746.984503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204746.984506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204746.984509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204747.176477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204747.176484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204747.176490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204747.176491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204747.176494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204747.176495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204747.176498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204747.176499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204747.176561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204747.176564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204747.176567:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204747.944464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204747.944471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204747.944472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204747.944477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204747.944480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204747.944483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204747.944485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204747.944487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204747.944489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204748.008518:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204748.008533:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204748.008533:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204748.008538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204748.008540:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204748.008544:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204748.008546:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204748.008549:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204748.008552:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204748.008554:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204748.008558:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204748.008559:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204748.008569:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204748.008573:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204748.008577:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204748.008578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204748.008586:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204748.008591:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204748.200477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204748.200482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204748.200486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204748.200487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204748.200489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204748.200490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204748.200493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204748.200494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204748.200508:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204748.200510:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204748.200512:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204748.968470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204748.968476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204748.968478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204748.968483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204748.968486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204748.968489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204748.968491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204748.968492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204748.968494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204749.032481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204749.032486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204749.032488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204749.032494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204749.032495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204749.032501:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204749.032505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204749.032508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204749.032510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204749.032519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204749.032519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204749.032524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204749.032528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204749.032528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204749.032544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204749.032545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204749.032550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204749.032555:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204749.224538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204749.224554:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204749.224557:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204749.224562:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224566:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204749.224569:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224571:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224574:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224579:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204749.224580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204749.224584:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204749.224586:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204749.224652:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204749.224656:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204749.224659:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204749.544478:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300876289, at 4300876800 00000100:00000010:1.0:1788204749.544503:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000040:0.0:1788204749.544592:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300876289, at 4300876800 00000100:00000010:0.0:1788204749.544598:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000001:1.0:1788204749.992471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204749.992478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204749.992479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204749.992484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204749.992487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204749.992489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204749.992492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204749.992494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204749.992497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204750.056456:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204750.056459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788204750.056465:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204750.056467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204750.056468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204750.056468:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204750.056472:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204750.056476:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204750.056477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204750.056479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204750.056481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204750.056482:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6204 00000100:00100000:0.0:1788204750.056489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204750.056489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204750.056497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1788204750.056497:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204750.056498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204750.056505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204750.056505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204750.056507:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204750.056512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204750.056515:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204750.056517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204750.056521:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204750.056524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204750.056524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056527:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788204750.056528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204750.056532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.056534:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056539:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204750.056544:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204750.056546:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204750.056548:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204750.056551:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204750.056554:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204750.056556:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204750.056559:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056561:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.056562:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056564:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204750.056566:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204750.056568:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056570:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204750.056572:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204750.056574:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204750.056577:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056578:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204750.056580:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204750.056589:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204750.056593:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204750.056597:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204750.056599:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056602:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204750.056604:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204750.056609:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204750.056618:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788204750.056625:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204750.056634:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205893760/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204750.056643:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204750.056647:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.056650:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205893760/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204750.056668:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204750.056670:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.056673:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204750.056674:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204750.056677:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204750.056680:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204750.056682:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6204 00000400:00000200:2.0:1788204750.056687:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204750.056692:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204750.056695:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204750.056699:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056702:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.056706:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056710:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204750.056715:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204750.056718:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204750.056722:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204750.056725:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204750.056729:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204750.056730:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204750.056734:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056737:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.056740:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056744:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204750.056748:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204750.056754:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056759:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204750.056764:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204750.056769:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204750.056772:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056776:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204750.056781:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204750.056785:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056789:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204750.056797:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204750.056802:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204750.056808:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204750.056812:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.056817:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204750.056820:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204750.056823:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204750.056832:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788204750.056839:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204750.056860:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205893888/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204750.056871:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204750.056874:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.056881:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205893888/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204750.056892:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204750.056896:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.056901:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204750.056903:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204750.056904:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204750.056908:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204750.056912:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6204 00000400:00000200:2.0:1788204750.056920:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204750.056924:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204750.056928:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204750.056942:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056947:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.056952:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056955:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204750.056960:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204750.056964:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204750.056966:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204750.056969:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204750.056974:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204750.056978:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204750.056986:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056989:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.056993:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.056996:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204750.056999:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204750.057004:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057009:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204750.057014:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204750.057018:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204750.057021:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057025:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204750.057027:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204750.057032:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057037:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204750.057043:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204750.057047:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204750.057049:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204750.057051:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057056:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204750.057059:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204750.057063:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204750.057078:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204750.057084:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204750.057094:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205894016/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204750.057103:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204750.057110:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:3.1:1788204750.057111:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788204750.057117:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.057126:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205894016/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204750.057140:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204750.057145:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.057150:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204750.057153:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204750.057157:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204750.057161:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204750.057167:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6204 00000400:00000200:2.0:1788204750.057177:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204750.057184:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204750.057189:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204750.057204:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057210:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.057216:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057221:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204750.057226:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204750.057230:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204750.057233:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204750.057236:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204750.057241:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204750.057247:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204750.057255:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057259:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204750.057262:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057265:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204750.057267:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204750.057273:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204750.057276:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204750.057280:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204750.057286:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204750.057291:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057295:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204750.057301:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204750.057307:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057313:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204750.057320:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204750.057327:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204750.057333:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204750.057336:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.057340:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204750.057344:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204750.057348:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204750.057355:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204750.057362:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204750.057397:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205894144/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204750.057414:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204750.057420:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.057430:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205894144/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204750.057448:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204750.057456:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6215) 00000800:00000200:3.1:1788204750.057707:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204750.057722:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204750.058042:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204750.058045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204750.058329:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204750.058332:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00010000:00000001:1.0:1788204750.248536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204750.248543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204750.248550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204750.248551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204750.248554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204750.248556:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204750.248558:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204750.248559:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204750.248574:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204750.248576:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204750.248578:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204751.016467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204751.016473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204751.016475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204751.016480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204751.016483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204751.016486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204751.016487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204751.016489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204751.016492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204751.080495:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204751.080495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204751.080505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204751.080505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204751.080510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204751.080511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204751.080518:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204751.080519:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204751.080524:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204751.080525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204751.080529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204751.080530:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204751.080537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204751.080541:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204751.080543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204751.080545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204751.080549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204751.080553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204751.272496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204751.272503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204751.272508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204751.272509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204751.272512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204751.272513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204751.272517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204751.272519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204751.272588:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204751.272592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204751.272595:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204752.040477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204752.040484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204752.040485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204752.040489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204752.040492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204752.040495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204752.040497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204752.040499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204752.040500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204752.104466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204752.104474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204752.104476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204752.104482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204752.104487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204752.104491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204752.104494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204752.104498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204752.104501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204752.104538:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204752.104542:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204752.104544:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204752.104546:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204752.104548:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204752.104550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204752.104552:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204752.104554:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204752.104556:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204752.296531:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204752.296534:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204752.296537:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296542:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204752.296544:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296546:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296548:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204752.296554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204752.296559:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204752.296560:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204752.296561:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296563:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204752.296564:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296565:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296567:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204752.296572:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204752.296573:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204752.296575:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296576:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204752.296577:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296579:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204752.296581:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204752.296597:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204752.296599:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204752.296602:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204753.064458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204753.064469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204753.064471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204753.064476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204753.064478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204753.064481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204753.064483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204753.064485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204753.064486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204753.128489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204753.128489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204753.128496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204753.128497:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204753.128499:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204753.128500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204753.128505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204753.128509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204753.128511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204753.128514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204753.128517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204753.128518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204753.128520:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204753.128523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204753.128528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204753.128528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204753.128531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204753.128534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204753.320500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204753.320507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204753.320514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204753.320515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204753.320517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204753.320518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204753.320520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204753.320521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204753.320579:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204753.320583:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204753.320586:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204754.088430:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204754.088435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204754.088437:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204754.088441:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204754.088443:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204754.088446:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204754.088448:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204754.088449:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204754.088452:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204754.152487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204754.152493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204754.152495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204754.152499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204754.152501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204754.152505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204754.152508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204754.152510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204754.152513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204754.152641:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204754.152647:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204754.152648:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204754.152653:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204754.152656:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204754.152660:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204754.152662:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204754.152664:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204754.152667:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204754.344499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204754.344505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204754.344511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204754.344513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204754.344517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204754.344519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204754.344522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204754.344523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204754.344544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204754.344546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204754.344549:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204754.664473:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300881409, at 4300881920 00000100:00000040:1.0:1788204754.664480:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300881409, at 4300881920 00000100:00000010:0.0:1788204754.664490:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000010:1.0:1788204754.664498:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000001:1.0:1788204755.112470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204755.112476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204755.112478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204755.112482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204755.112486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204755.112489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204755.112492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204755.112494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204755.112497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204755.176485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204755.176493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204755.176496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204755.176502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204755.176503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204755.176506:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204755.176509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204755.176510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204755.176516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204755.176518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204755.176521:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1788204755.176523:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204755.176524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204755.176527:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204755.176530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204755.176534:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204755.176534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204755.176535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204755.176546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176547:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204755.176548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204755.176553:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204755.176557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204755.176561:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6210 00000400:00000200:0.0:1788204755.176576:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204755.176586:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204755.176591:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204755.176595:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176600:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.176602:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176605:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204755.176610:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204755.176613:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204755.176616:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204755.176620:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204755.176622:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204755.176624:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204755.176626:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176628:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.176630:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176631:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204755.176634:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204755.176636:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176638:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204755.176640:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204755.176644:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204755.176647:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176649:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204755.176651:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204755.176664:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204755.176668:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204755.176672:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204755.176674:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176678:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204755.176690:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204755.176695:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204755.176707:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204755.176714:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204755.176724:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205894272/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204755.176732:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204755.176736:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204755.176739:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205894272/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204755.176770:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204755.176777:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204755.176786:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204755.176788:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204755.176789:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204755.176793:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204755.176797:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6210 00000400:00000200:0.0:1788204755.176802:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204755.176806:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204755.176808:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204755.176812:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176821:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.176826:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176833:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204755.176836:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204755.176839:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204755.176843:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204755.176847:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204755.176852:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204755.176857:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204755.176864:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176868:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.176874:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176879:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204755.176883:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204755.176891:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204755.176896:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204755.176899:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204755.176908:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204755.176912:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176914:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204755.176920:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204755.176925:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176931:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204755.176939:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204755.176944:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204755.176949:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204755.176951:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.176954:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204755.176958:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204755.176960:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204755.176966:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204755.176972:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204755.176979:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205894400/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204755.176992:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204755.176996:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204755.177006:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205894400/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204755.177022:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204755.177027:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204755.177033:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204755.177036:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204755.177037:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204755.177044:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204755.177049:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6210 00000400:00000200:0.0:1788204755.177054:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204755.177057:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204755.177060:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204755.177066:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177070:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.177075:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177080:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204755.177083:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204755.177086:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204755.177089:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204755.177094:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204755.177097:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204755.177101:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204755.177107:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177110:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.177112:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177115:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204755.177118:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204755.177122:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177124:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204755.177125:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204755.177129:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204755.177132:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177135:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204755.177137:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204755.177139:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177141:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204755.177145:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204755.177148:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204755.177151:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204755.177154:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177156:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204755.177157:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204755.177159:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204755.177164:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204755.177168:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204755.177172:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205894528/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204755.177179:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204755.177183:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204755.177187:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205894528/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204755.177195:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204755.177198:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204755.177201:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204755.177202:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204755.177205:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204755.177207:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1788204755.177208:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1788204755.177210:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204755.177217:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6210 00000400:00000200:0.0:1788204755.177224:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204755.177228:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204755.177231:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204755.177239:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177245:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.177250:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177256:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204755.177260:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204755.177262:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204755.177266:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204755.177271:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204755.177277:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204755.177283:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204755.177291:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177295:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204755.177302:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177307:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204755.177313:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204755.177320:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204755.177324:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204755.177327:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204755.177332:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204755.177338:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177343:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204755.177347:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204755.177350:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177355:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204755.177359:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204755.177392:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204755.177395:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204755.177397:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204755.177398:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204755.177399:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204755.177401:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204755.177406:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204755.177410:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204755.177416:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205894656/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204755.177423:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204755.177427:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204755.177430:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205894656/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204755.177440:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204755.177445:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6220) 00000800:00000200:3.1:1788204755.177744:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204755.177746:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204755.178051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204755.178053:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204755.178247:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204755.178250:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00010000:00000001:1.0:1788204755.368510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204755.368520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204755.368529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204755.368530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204755.368534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204755.368535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204755.368539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204755.368540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204755.368600:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204755.368605:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204755.368609:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204756.136510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204756.136521:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204756.136524:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204756.136531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204756.136535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204756.136539:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204756.136541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204756.136544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204756.136546:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.200503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204756.200504:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204756.200513:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204756.200513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204756.200519:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.200520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204756.200530:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204756.200530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204756.200541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204756.200542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204756.200550:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204756.200550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204756.200560:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204756.200562:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204756.200565:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.200566:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204756.200572:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204756.200575:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204756.392528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204756.392535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204756.392539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204756.392540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204756.392542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204756.392543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204756.392545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204756.392546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204756.392562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204756.392564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204756.392567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204757.160510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204757.160519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204757.160522:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204757.160528:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204757.160532:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204757.160537:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204757.160540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204757.160542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204757.160546:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204757.224487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204757.224494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204757.224496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204757.224502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204757.224503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204757.224507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204757.224514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204757.224515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204757.224519:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204757.224522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204757.224524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204757.224527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204757.224528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204757.224531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204757.224536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204757.224537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204757.224541:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204757.224545:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204757.416496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204757.416503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204757.416508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204757.416509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204757.416512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204757.416513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204757.416515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204757.416517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204757.416571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204757.416575:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204757.416579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204758.184475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204758.184482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204758.184484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204758.184490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204758.184493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204758.184496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204758.184499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204758.184501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204758.184504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204758.248456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204758.248457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204758.248463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204758.248464:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204758.248466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204758.248467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204758.248473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204758.248474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204758.248479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204758.248480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204758.248483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204758.248484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204758.248488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204758.248491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204758.248494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204758.248496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204758.248500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204758.248502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204758.440462:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204758.440467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204758.440471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204758.440472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204758.440474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204758.440475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204758.440477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204758.440478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204758.440489:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204758.440491:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204758.440493:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204759.208432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204759.208437:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204759.208438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204759.208442:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204759.208444:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204759.208446:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204759.208448:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204759.208450:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204759.208452:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.272449:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204759.272453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204759.272456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204759.272458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204759.272458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204759.272462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204759.272463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204759.272468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204759.272468:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204759.272472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204759.272473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204759.272475:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204759.272477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204759.272478:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204759.272483:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204759.272484:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204759.272491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204759.272495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204759.464504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204759.464510:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204759.464513:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204759.464515:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464518:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204759.464519:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464521:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464522:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204759.464525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204759.464527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204759.464528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204759.464596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204759.464599:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204759.464602:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204759.784507:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300886530, at 4300887040 00000100:00000040:0.0:1788204759.784514:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300886529, at 4300887040 00000100:00000010:1.0:1788204759.784530:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000010:0.0:1788204759.784531:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000001:1.0:1788204760.232507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204760.232516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204760.232518:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204760.232525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204760.232527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204760.232530:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204760.232532:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204760.232534:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204760.232536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204760.296469:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204760.296469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204760.296469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204760.296477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204760.296477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204760.296478:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204760.296481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204760.296482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296483:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204760.296488:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204760.296489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204760.296489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204760.296495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204760.296496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204760.296497:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204760.296500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204760.296501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204760.296505:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6215 00000100:00000001:2.0:1788204760.296505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204760.296511:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204760.296515:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204760.296518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204760.296518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204760.296522:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204760.296523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204760.296525:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204760.296529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.296530:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296535:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.296537:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296539:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204760.296543:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204760.296545:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204760.296547:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204760.296550:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204760.296552:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204760.296554:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204760.296556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296558:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.296560:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296561:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204760.296563:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204760.296565:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296567:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204760.296569:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204760.296571:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204760.296573:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296575:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204760.296576:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204760.296584:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204760.296586:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204760.296589:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204760.296591:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296593:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204760.296595:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204760.296599:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204760.296606:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204760.296611:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204760.296618:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205894784/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204760.296626:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204760.296630:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.296633:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205894784/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204760.296655:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204760.296658:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204760.296661:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204760.296663:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204760.296664:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204760.296667:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204760.296701:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6215 00000400:00000200:0.0:1788204760.296706:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204760.296710:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204760.296714:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204760.296716:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296718:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.296721:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296724:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204760.296725:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204760.296726:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204760.296727:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204760.296730:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204760.296731:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204760.296733:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204760.296736:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296739:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.296742:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296748:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204760.296753:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204760.296759:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296764:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204760.296766:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204760.296775:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204760.296783:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296790:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204760.296798:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204760.296804:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296810:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204760.296818:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204760.296824:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204760.296829:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204760.296834:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.296837:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204760.296839:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204760.296842:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204760.296849:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204760.296854:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204760.296863:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205894912/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204760.296877:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204760.296884:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.296890:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205894912/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204760.296928:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204760.296932:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204760.296934:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204760.296935:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204760.296938:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204760.296947:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204760.296951:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6215 00000400:00000200:0.0:1788204760.296955:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204760.296958:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204760.296959:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204760.296962:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296965:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.296967:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.296979:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204760.296986:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204760.296988:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204760.296991:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204760.296995:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204760.296999:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204760.297005:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204760.297029:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297038:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.297046:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297051:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204760.297060:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204760.297063:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297065:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204760.297073:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204760.297083:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204760.297087:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297089:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204760.297091:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204760.297095:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297101:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204760.297105:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204760.297107:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204760.297113:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204760.297118:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297127:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204760.297128:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204760.297131:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204760.297136:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204760.297142:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204760.297148:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205895040/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204760.297166:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204760.297169:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.297174:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205895040/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204760.297197:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204760.297201:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204760.297205:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204760.297209:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204760.297211:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204760.297214:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204760.297220:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6215 00000400:00000200:0.0:1788204760.297243:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204760.297253:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204760.297258:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204760.297262:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297266:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.297271:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297275:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204760.297282:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204760.297286:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204760.297291:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204760.297295:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204760.297302:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788204760.297308:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:0.0:1788204760.297309:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788204760.297319:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000100:00000040:0.0:1788204760.297321:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297327:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204760.297331:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297335:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204760.297339:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204760.297341:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204760.297343:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204760.297346:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204760.297350:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204760.297354:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297359:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204760.297361:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204760.297391:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297392:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204760.297395:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204760.297397:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204760.297400:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204760.297401:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204760.297402:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204760.297405:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204760.297407:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204760.297410:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204760.297415:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204760.297423:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205895168/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204760.297446:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204760.297450:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.297464:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205895168/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204760.297476:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204760.297482:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6225) 00000800:00000200:3.1:1788204760.297934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204760.297938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204760.298252:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204760.298255:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204760.298537:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204760.298539:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00010000:00000001:1.0:1788204760.488590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204760.488596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204760.488601:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204760.488609:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204760.488612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204760.488613:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204760.488615:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204760.488616:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204760.488635:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204760.488637:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204760.488639:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204761.256454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204761.256466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204761.256469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204761.256477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204761.256482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204761.256486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204761.256489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204761.256491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204761.256494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204761.320491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204761.320496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204761.320498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204761.320502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204761.320504:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204761.320508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204761.320510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204761.320517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204761.320518:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204761.320525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204761.320526:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204761.320531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204761.320532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204761.320536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204761.320540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204761.320543:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204761.320547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204761.320551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204761.512480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204761.512488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204761.512495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204761.512496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204761.512500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204761.512501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204761.512505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204761.512506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204761.512580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204761.512584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204761.512588:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204762.280509:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204762.280518:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204762.280520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204762.280526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204762.280529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204762.280534:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204762.280536:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204762.280538:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204762.280541:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204762.344496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204762.344505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204762.344509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204762.344516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204762.344523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204762.344528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204762.344542:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204762.344547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204762.344550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204762.536532:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204762.536534:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204762.536537:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536540:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204762.536542:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536544:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536545:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204762.536549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204762.536553:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204762.536554:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204762.536554:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536555:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204762.536556:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536557:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536559:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204762.536562:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204762.536562:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204762.536563:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536564:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204762.536565:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536566:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204762.536568:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204762.536582:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204762.536584:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204762.536586:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204763.304436:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204763.304442:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204763.304444:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204763.304448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204763.304451:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204763.304454:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204763.304456:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204763.304459:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204763.304461:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204763.368479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204763.368485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204763.368490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204763.368497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204763.368501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204763.368506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204763.368512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204763.368515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204763.368517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204763.560451:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204763.560457:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204763.560462:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204763.560463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204763.560466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204763.560467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204763.560470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204763.560471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204763.560528:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204763.560532:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204763.560536:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204764.328495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204764.328501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204764.328502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204764.328507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204764.328510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204764.328513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204764.328515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204764.328517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204764.328519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204764.392439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204764.392445:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204764.392447:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204764.392453:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204764.392457:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204764.392461:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204764.392464:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204764.392467:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204764.392470:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204764.584518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204764.584523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204764.584527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204764.584528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204764.584530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204764.584531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204764.584533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204764.584534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204764.584548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204764.584550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204764.584552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204764.904448:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300891650, at 4300892160 00000100:00000010:1.0:1788204764.904462:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000040:0.0:1788204764.904471:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300891650, at 4300892160 00000100:00000010:0.0:1788204764.904480:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000001:1.0:1788204765.352479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204765.352488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204765.352491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204765.352496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204765.352500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204765.352511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204765.352514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204765.352515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204765.352518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.416460:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204765.416461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204765.416468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204765.416473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.416475:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1788204765.416479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204765.416481:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204765.416485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204765.416488:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204765.416491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204765.416495:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204765.416497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204765.416500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204765.416502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.416515:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6220 00000400:00000200:0.0:1788204765.416526:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204765.416534:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204765.416538:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204765.416541:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416545:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.416548:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416550:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204765.416554:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204765.416557:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204765.416568:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204765.416572:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204765.416575:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204765.416577:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204765.416612:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416616:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.416618:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416621:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204765.416624:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204765.416628:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416631:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204765.416635:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204765.416638:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204765.416642:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.416644:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204765.416646:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204765.416656:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204765.416660:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204765.416672:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204765.416674:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.416677:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204765.416680:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204765.416685:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204765.416694:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204765.416702:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204765.416713:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205895296/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204765.416724:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204765.416729:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204765.416735:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205895296/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204765.416765:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.416768:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204765.416772:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204765.416774:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204765.416776:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204765.416779:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204765.416784:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6220 00000400:00000200:0.0:1788204765.416790:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204765.416797:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204765.416804:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204765.416812:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416818:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.416824:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416829:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204765.416834:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204765.416837:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204765.416841:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204765.416844:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204765.416849:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204765.416852:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204765.416857:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416862:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.416867:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416872:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204765.416876:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204765.416885:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204765.416890:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204765.416895:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204765.416901:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204765.416907:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.416911:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204765.416917:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204765.416924:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.416931:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204765.416938:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204765.416944:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204765.416950:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204765.416952:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.416954:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204765.416959:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204765.416964:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204765.416974:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204765.416980:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204765.416992:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205895424/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204765.417009:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204765.417013:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204765.417023:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205895424/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204765.417038:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.417043:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204765.417050:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204765.417054:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204765.417058:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204765.417064:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204765.417071:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6220 00000400:00000200:0.0:1788204765.417076:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204765.417083:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204765.417087:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204765.417093:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417097:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.417099:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417104:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204765.417107:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204765.417109:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204765.417112:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204765.417115:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204765.417117:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204765.417118:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204765.417124:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417126:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.417131:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417135:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204765.417137:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204765.417143:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417148:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204765.417149:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204765.417152:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204765.417155:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417160:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204765.417162:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204765.417164:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417167:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204765.417172:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204765.417175:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204765.417178:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204765.417181:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417184:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204765.417187:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204765.417189:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204765.417193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000010:0.0:1788204765.417195:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000800:00000200:3.1:1788204765.417197:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1788204765.417200:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204765.417210:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205895552/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204765.417219:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204765.417221:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204765.417223:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205895552/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204765.417229:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204765.417233:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204765.417238:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204765.417241:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204765.417243:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204765.417247:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204765.417252:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6220 00000400:00000200:0.0:1788204765.417259:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204765.417263:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204765.417266:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204765.417271:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417275:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.417279:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417281:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204765.417285:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204765.417287:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204765.417290:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204765.417292:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204765.417295:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204765.417300:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204765.417313:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417317:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204765.417320:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417321:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204765.417326:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204765.417331:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204765.417333:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204765.417335:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204765.417339:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204765.417342:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417344:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204765.417349:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204765.417354:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417358:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204765.417396:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204765.417402:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204765.417407:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204765.417410:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204765.417414:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204765.417417:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204765.417420:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204765.417425:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204765.417431:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204765.417439:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205895680/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204765.417466:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204765.417468:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204765.417476:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205895680/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204765.417492:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204765.417498:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6230) 00000800:00000200:3.1:1788204765.417689:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204765.417704:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204765.417934:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204765.417936:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204765.418161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204765.418163:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00010000:00000001:1.0:1788204765.608451:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204765.608458:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204765.608464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204765.608466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204765.608469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204765.608471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204765.608475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204765.608476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204765.608542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204765.608546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204765.608548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204766.376474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204766.376481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204766.376483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204766.376487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204766.376490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204766.376493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204766.376495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204766.376498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204766.376502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204766.440467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204766.440475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204766.440478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204766.440485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204766.440488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204766.440491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204766.440493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204766.440497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204766.440500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204766.632475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204766.632481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204766.632485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204766.632487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204766.632491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204766.632492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204766.632496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204766.632497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204766.632516:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204766.632520:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204766.632523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204767.400464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204767.400470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204767.400472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204767.400476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204767.400479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204767.400482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204767.400484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204767.400486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204767.400488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204767.464460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204767.464468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204767.464471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204767.464480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204767.464487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204767.464493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204767.464509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204767.464513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204767.464518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204767.656460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204767.656469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204767.656475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204767.656477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204767.656480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204767.656481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204767.656484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204767.656485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204767.656569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204767.656573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204767.656577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204768.424504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204768.424513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204768.424516:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204768.424522:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204768.424527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204768.424564:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204768.424568:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204768.424571:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204768.424575:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204768.488477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204768.488485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204768.488487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204768.488493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204768.488496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204768.488501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204768.488504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204768.488506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204768.488510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204768.680515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204768.680519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204768.680537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204768.680538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204768.680541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204768.680542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204768.680545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204768.680545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204768.680559:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204768.680561:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204768.680564:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204769.448493:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204769.448502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204769.448504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204769.448509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204769.448525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204769.448530:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204769.448532:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204769.448535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204769.448538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204769.512468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204769.512478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204769.512482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204769.512490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204769.512498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204769.512506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204769.512541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204769.512545:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204769.512550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204769.704548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204769.704561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204769.704565:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204769.704577:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704581:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204769.704588:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704590:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704593:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704598:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204769.704599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204769.704604:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204769.704605:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204769.704678:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204769.704683:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204769.704685:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204770.024439:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300896769, at 4300897280 00000100:00000040:0.0:1788204770.024440:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300896769, at 4300897280 00000100:00000010:0.0:1788204770.024459:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000010:1.0:1788204770.024464:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000001:1.0:1788204770.472527:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204770.472537:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204770.472540:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204770.472546:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204770.472551:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204770.472556:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204770.472559:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204770.472562:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204770.472567:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204770.536514:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204770.536525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204770.536530:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204770.536540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204770.536547:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204770.536554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204770.536561:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204770.536565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204770.536569:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.536901:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204770.536912:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204770.536919:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204770.536937:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.536962:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204770.536966:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.536973:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204770.536978:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204770.536990:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204770.537007:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205895808:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204770.537015:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204770.537024:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204770.537029:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.537037:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204770.537061:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204770.537068:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204770.537083:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204770.537091:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204770.537104:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204770.537114:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205895808, portal 25 00000100:00000001:2.0:1788204770.537134:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204770.537140:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000040:2.0:1788204770.537162:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537184:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204770.537193:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204770.537200:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204770.537215:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205895808, offset 0 00000400:00000200:2.0:1788204770.537232:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204770.537249:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.537263:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204770.537273:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204770.537288:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204770.537299:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204770.537308:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204770.537317:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58854:58854:256:4294967295] 192.168.201.151@tcp LPNI seq info [58834:58834:8:4294967295] 00000400:00000200:2.0:1788204770.537349:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.537428:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204770.537442:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.537454:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204770.537470:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204770.537483:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204770.537515:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204770.537525:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204770.537542:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.537551:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204770.537567:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204770.537580:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204770.537590:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204770.537596:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204770.537603:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204770.537610:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537664:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.537676:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537704:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.537711:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204770.537722:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204770.537742:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204770.537749:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300901890 00000100:00000010:2.0:1788204770.537759:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1de0. 00000100:00000001:2.0:1788204770.537770:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204770.537782:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300901890 00000100:00000010:2.0:1788204770.537785:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1000. 00000100:00000001:2.0:1788204770.537789:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204770.537794:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204770.537797:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.537799:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204770.537804:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204770/real 1788204770] req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204770.537812:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204770.537813:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204770.537815:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204770.537817:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204770.537819:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204770.537823:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204770.537826:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204770.537835:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537843:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204770.537845:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537848:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.537850:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204770.537854:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537862:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204770.537868:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537876:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204770.537879:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537881:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.537882:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537885:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204770.537887:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204770.537891:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204770.537896:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204770.537898:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204770.537902:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204770.537911:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205895808:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204770.537916:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537918:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.537922:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205895808/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537929:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204770.537932:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204770.537935:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204770.537938:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204770.537941:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204770.537942:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204770.537944:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204770.537946:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204770.537949:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204770.537951:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204770.537952:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:2.0:1788204770.537955:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204770.537956:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537957:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.537963:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204770.537964:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.537966:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204770.537967:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204770.537969:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204770.537972:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.537977:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204770.537978:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.537980:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204770.537982:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204770.537983:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204770.537985:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205895936:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204770.537987:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204770.537989:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204770.537990:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.537991:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204770.537994:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204770.537995:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204770.537998:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204770.538000:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204770.538002:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204770.538005:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205895936, portal 10 00000100:00000001:2.0:1788204770.538006:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204770.538007:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:2.0:1788204770.538012:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538017:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204770.538019:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204770.538020:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204770.538022:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205895936, offset 0 00000400:00000200:2.0:1788204770.538024:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538030:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204770.538032:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204770.538034:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204770.538036:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204770.538038:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538040:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58855:58855:256:4294967295] 192.168.201.151@tcp LPNI seq info [58835:58835:8:4294967295] 00000400:00000200:2.0:1788204770.538043:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538048:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204770.538051:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538053:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204770.538056:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204770.538058:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204770.538062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204770.538064:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204770.538067:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538068:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204770.538070:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204770.538072:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204770.538074:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204770.538076:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204770.538077:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204770.538080:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538085:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538087:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538096:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538097:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204770.538099:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204770.538102:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204770.538104:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204770.538105:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300897947 00000100:00000001:2.0:1788204770.538107:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538109:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300897947 00000100:00000001:2.0:1788204770.538110:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538111:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204770.538113:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538114:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204770.538116:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204770/real 1788204770] req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204770.538121:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204770.538122:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204770.538123:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204770.538124:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204770.538125:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204770.538126:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204770.538128:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204770.538133:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538137:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204770.538138:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538140:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538141:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204770.538143:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538148:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204770.538151:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538155:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204770.538156:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538157:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538158:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538159:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204770.538161:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204770.538163:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204770.538166:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204770.538169:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204770.538172:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204770.538180:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205895936:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204770.538183:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204770.538187:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538195:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204770.538196:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.538199:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204770.538201:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204770.538203:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204770.538206:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896064:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204770.538209:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204770.538211:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204770.538213:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.538215:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204770.538219:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204770.538222:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204770.538225:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204770.538228:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204770.538232:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204770.538235:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896064, portal 4 00000100:00000001:2.0:1788204770.538237:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204770.538239:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:2.0:1788204770.538245:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538253:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204770.538256:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204770.538258:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204770.538261:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205896064, offset 0 00000400:00000200:2.0:1788204770.538266:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538270:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538274:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204770.538277:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204770.538280:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204770.538283:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204770.538285:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538289:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58856:58856:256:4294967295] 192.168.201.151@tcp LPNI seq info [58836:58836:8:4294967295] 00000400:00000200:2.0:1788204770.538293:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538300:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204770.538306:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538309:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204770.538314:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204770.538316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204770.538321:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204770.538324:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204770.538328:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538331:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204770.538334:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204770.538338:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204770.538341:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204770.538342:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204770.538345:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204770.538349:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538358:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538361:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538389:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538390:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204770.538393:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204770.538395:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204770.538398:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204770.538399:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300897834 00000100:00000001:2.0:1788204770.538401:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538402:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300897834 00000100:00000001:2.0:1788204770.538404:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538405:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204770.538406:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538408:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204770.538410:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204770/real 1788204770] req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204770.538416:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204770.538417:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204770.538418:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204770.538420:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204770.538421:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204770.538423:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204770.538425:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204770.538430:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538434:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204770.538435:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538436:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538437:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204770.538439:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538443:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204770.538445:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538450:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204770.538452:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538453:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538453:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538455:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204770.538456:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204770.538458:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204770.538460:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204770.538461:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204770.538464:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204770.538469:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896064:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204770.538471:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204770.538474:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538479:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204770.538480:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.538482:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204770.538483:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204770.538484:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204770.538486:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204770.538488:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204770.538489:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204770.538490:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204770.538507:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204770.538511:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204770.538512:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204770.538515:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204770.538516:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000400:00000010:2.0:1788204770.538519:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000200:2.0:1788204770.538521:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896192, portal 4 00000100:00000001:2.0:1788204770.538522:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204770.538523:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:2.0:1788204770.538528:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538532:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204770.538533:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204770.538535:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000100:00000200:2.0:1788204770.538536:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205896192, offset 0 00000400:00000200:2.0:1788204770.538539:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538541:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538544:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204770.538546:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204770.538548:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204770.538549:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204770.538551:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204770.538554:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58857:58857:256:4294967295] 192.168.201.151@tcp LPNI seq info [58837:58837:8:4294967295] 00000400:00000200:2.0:1788204770.538556:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204770.538561:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204770.538564:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538566:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204770.538569:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204770.538571:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1788204770.538574:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204770.538575:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1788204770.538578:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204770.538579:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204770.538581:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204770.538584:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204770.538586:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204770.538587:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204770.538589:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204770.538591:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538596:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538598:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538602:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538603:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204770.538605:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204770.538607:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204770.538609:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204770.538610:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300897840 00000100:00000001:2.0:1788204770.538612:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538613:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300897840 00000100:00000001:2.0:1788204770.538615:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300901890 00000100:00000001:2.0:1788204770.538616:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204770.538617:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538619:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204770.538621:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204770/real 1788204770] req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204770.538626:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204770.538627:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204770.538628:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa9a0. 00000400:00000200:2.0:1788204770.538629:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85990 00000400:00000010:2.0:1788204770.538631:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85990. 00000100:00000001:2.0:1788204770.538633:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204770.538636:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204770.538645:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538652:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204770.538654:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538655:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538657:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204770.538661:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538668:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204770.538672:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538680:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204770.538683:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538685:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204770.538686:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538688:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204770.538690:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204770.538693:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204770.538698:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204770.538700:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204770.538705:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204770.538713:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204770.538717:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538719:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538723:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205895936/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538730:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204770.538731:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204770.538733:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e000. 02000000:00000001:2.0:1788204770.538736:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204770.538738:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204770.538739:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204770.538741:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204770.538743:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204770.538745:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204770.538747:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204770.538748:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:2.0:1788204770.538751:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204770.538752:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538754:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538757:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205896064/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538763:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204770.538764:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204770.538766:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204770.538769:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204770.538770:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204770.538771:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204770.538773:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204770.538775:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204770.538777:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204770.538778:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204770.538779:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:2.0:1788204770.538781:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204770.538783:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538784:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204770.538787:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205896192/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204770.538793:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204770.538794:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204770.538795:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e400. 02000000:00000001:2.0:1788204770.538797:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204770.538799:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204770.538800:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204770.538802:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204770.538803:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204770.538806:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204770.538807:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204770.538808:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:2.0:1788204770.538810:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204770.538811:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538813:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204770.538817:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204770.538818:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204770.538820:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204770.538822:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204770.728565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204770.728581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204770.728588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204770.728589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204770.728592:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204770.728593:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204770.728596:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204770.728597:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204770.728617:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204770.728620:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204770.728623:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204771.496499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204771.496507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204771.496509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204771.496518:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204771.496521:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204771.496525:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204771.496528:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204771.496530:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204771.496533:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204771.560444:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204771.560456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204771.560460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204771.560470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204771.560487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204771.560492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204771.560497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204771.560500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204771.560504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204771.752531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204771.752540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204771.752546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204771.752547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204771.752549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204771.752550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204771.752553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204771.752554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204771.752625:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204771.752630:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204771.752633:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204772.520499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204772.520507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204772.520509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204772.520513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204772.520516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204772.520519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204772.520521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204772.520522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204772.520525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204772.584483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204772.584489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204772.584492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204772.584500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204772.584507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204772.584513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204772.584520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204772.584522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204772.584525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204772.776553:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204772.776556:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204772.776559:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776564:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204772.776566:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776567:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776570:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204772.776575:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204772.776580:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204772.776581:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204772.776583:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776584:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204772.776585:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776587:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776589:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776592:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204772.776593:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204772.776594:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204772.776596:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776598:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204772.776599:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776601:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204772.776603:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204772.776637:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204772.776640:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204772.776642:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204773.544476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204773.544482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204773.544484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204773.544488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204773.544490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204773.544493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204773.544495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204773.544496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204773.544498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204773.608506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204773.608513:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204773.608515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204773.608524:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204773.608529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204773.608534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204773.608538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204773.608542:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204773.608545:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204773.800523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204773.800531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204773.800537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204773.800538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204773.800541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204773.800542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204773.800545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204773.800546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204773.800651:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204773.800658:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204773.800664:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204774.568473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204774.568481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204774.568483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204774.568488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204774.568491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204774.568495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204774.568498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204774.568500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204774.568503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204774.632444:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204774.632454:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204774.632457:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204774.632467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204774.632474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204774.632482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204774.632488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204774.632493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204774.632496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204774.824576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204774.824586:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204774.824593:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204774.824595:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204774.824598:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204774.824600:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204774.824604:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204774.824605:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204774.824632:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204774.824636:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204774.824639:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204775.144510:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300901890, at 4300902400 00000100:00000040:0.0:1788204775.144516:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300901890, at 4300902400 00000100:00000010:1.0:1788204775.144529:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000010:0.0:1788204775.144534:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000001:2.0:1788204775.144735:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204775.144740:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204775.144743:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204775.144748:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204775.144751:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204775.144756:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.144758:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204775.144762:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.144766:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204775.592500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204775.592509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204775.592511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204775.592517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204775.592520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204775.592524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204775.592527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204775.592530:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204775.592532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204775.656521:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204775.656533:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204775.656538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204775.656548:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204775.656556:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204775.656558:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204775.656562:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204775.656575:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204775.656576:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204775.656580:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204775.656582:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204775.656584:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204775.656594:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204775.656596:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204775.656606:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6230 00000400:00000200:0.0:1788204775.656623:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204775.656633:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204775.656638:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204775.656643:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656648:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.656651:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656654:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204775.656659:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204775.656661:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204775.656663:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204775.656668:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204775.656670:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204775.656671:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204775.656674:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656675:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.656676:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656678:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204775.656680:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204775.656682:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656684:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204775.656686:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204775.656690:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204775.656693:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.656694:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204775.656697:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204775.656707:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204775.656710:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204775.656713:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204775.656715:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.656725:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204775.656729:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204775.656734:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204775.656747:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204775.656755:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204775.656769:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204775.656780:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204775.656785:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204775.656789:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.656813:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204775.656816:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204775.656820:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204775.656821:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204775.656823:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204775.656825:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204775.656827:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6230 00000400:00000200:0.0:1788204775.656830:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204775.656832:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204775.656834:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204775.656836:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656837:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.656839:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656842:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204775.656844:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204775.656845:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204775.656847:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204775.656850:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204775.656853:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204775.656855:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204775.656859:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656862:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.656865:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204775.656866:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204775.656868:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204775.656874:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788204775.656875:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204775.656879:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788204775.656885:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204775.656890:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204775.656897:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1788204775.656899:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204775.656906:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204775.656913:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.656922:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204775.656931:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788204775.656931:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204775.656937:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788204775.656938:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.656949:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.656953:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1788204775.656955:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 02000000:00000001:2.0:1788204775.656959:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788204775.656960:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204775.656965:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204775.656971:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.656972:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.656975:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204775.656979:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204775.656982:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204775.656987:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205896320:192.168.201.151@tcp:250:kworker.0 02000000:00000010:0.0:1788204775.656989:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204775.657002:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657002:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204775.657008:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204775.657013:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204775.657017:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204775.657018:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204775.657031:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:0.0:1788204775.657034:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788204775.657036:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.657040:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204775.657046:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204775.657056:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204775.657070:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204775.657070:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa9a0. 00000100:00080000:0.0:1788204775.657075:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204775.657079:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204775.657080:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204775.657082:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1788204775.657083:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85990. 00000100:00000001:0.0:1788204775.657086:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204775.657091:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6230 00000100:00000200:2.0:1788204775.657096:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896320, portal 25 00000400:00000200:0.0:1788204775.657097:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204775.657109:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204775.657111:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788204775.657113:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204775.657115:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:0.0:1788204775.657121:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657125:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.657127:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657133:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:2.0:1788204775.657134:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204775.657138:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204775.657141:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204775.657146:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204775.657150:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204775.657157:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204775.657166:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204775.657173:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657180:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204775.657182:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204775.657188:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:2.0:1788204775.657192:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657199:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000010:2.0:1788204775.657201:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 00000020:00000040:0.0:1788204775.657208:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000200:2.0:1788204775.657209:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205896320, offset 0 00000100:00080000:0.0:1788204775.657219:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788204775.657229:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657230:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204775.657236:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204775.657241:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204775.657246:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657247:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204775.657254:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204775.657259:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:2.0:1788204775.657260:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00000001:0.0:1788204775.657269:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657273:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204775.657276:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657282:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000010:0.0:1788204775.657285:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204775.657290:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204775.657292:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204775.657293:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.657296:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204775.657298:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1788204775.657303:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1788204775.657304:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204775.657316:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000400:00000200:2.0:1788204775.657317:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657323:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657330:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58858:58858:256:4294967295] 192.168.201.151@tcp LPNI seq info [58838:58838:8:4294967295] 00000100:00100000:0.0:1788204775.657332:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204775.657361:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204775.657439:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204775.657448:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657449:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:0.0:1788204775.657462:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788204775.657469:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657488:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204775.657488:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000100:00080000:0.0:1788204775.657496:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1788204775.657505:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204775.657507:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204775.657512:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204775.657514:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1788204775.657521:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204775.657524:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204775.657531:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6230 00000400:00000200:0.0:1788204775.657538:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204775.657543:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204775.657544:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204775.657547:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204775.657554:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204775.657557:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204775.657560:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300907010 00000100:00000001:0.0:1788204775.657565:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204775.657571:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:2.0:1788204775.657571:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1960. 00000100:00000001:0.0:1788204775.657580:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204775.657583:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788204775.657585:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204775.657590:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204775.657593:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204775.657594:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300907010 00000020:00000040:0.0:1788204775.657605:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000010:2.0:1788204775.657605:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c10c0. 00000020:00000001:0.0:1788204775.657613:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204775.657616:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788204775.657621:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204775.657625:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788204775.657631:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204775.657633:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.657638:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204775.657642:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204775.657646:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204775.657649:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204775.657653:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204775.657660:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:2.0:1788204775.657663:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204775.657664:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204775.657669:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657669:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1788204775.657676:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204775.657683:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204775.657683:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788204775.657690:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657691:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204775.657693:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204775.657695:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204775.657700:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657702:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204775.657704:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204775.657709:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204775.657716:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000040:2.0:1788204775.657717:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204775.657720:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204775.657724:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.657730:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204775.657734:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204775.657738:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657741:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1788204775.657746:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:2.0:1788204775.657746:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.657752:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204775.657756:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.657757:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204775.657766:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204775.657773:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788204775.657778:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896448:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204775.657798:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204775.657801:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788204775.657803:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204775.657810:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.657811:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.657821:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204775.657827:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e000. 00000100:00000040:0.0:1788204775.657828:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204775.657834:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.657850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204775.657857:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aacb8. 00000100:00000001:0.0:1788204775.657863:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204775.657865:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85088. 00000100:00000200:2.0:1788204775.657873:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896448, portal 10 00000100:00000040:0.0:1788204775.657874:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6240) 00000100:00000001:2.0:1788204775.657877:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204775.657882:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204775.657893:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204775.657901:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204775.657907:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204775.657910:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85908. 00000100:00000200:2.0:1788204775.657913:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205896448, offset 0 00000400:00000200:2.0:1788204775.657919:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204775.657924:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.657929:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204775.657932:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204775.657935:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204775.657937:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204775.657940:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204775.657943:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58859:58859:255:4294967295] 192.168.201.151@tcp LPNI seq info [58839:58839:7:4294967295] 00000400:00000200:2.0:1788204775.657947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.657954:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204775.657959:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204775.657963:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4400. 00000800:00000200:2.0:1788204775.657967:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204775.657971:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204775.657975:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204775.657979:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204775.657982:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204775.657984:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300903067 00000100:00000001:2.0:1788204775.657987:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.657989:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300903067 00000100:00000001:2.0:1788204775.657991:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.657993:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204775.657995:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.657998:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204775.658002:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204775.658009:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204775.658011:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.658013:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204775.658015:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204775.658016:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204775.658019:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896576:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204775.658023:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204775.658025:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204775.658027:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.658028:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204775.658032:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204775.658034:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.658038:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204775.658040:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa1b8. 00000400:00000010:2.0:1788204775.658044:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85660. 00000100:00000200:2.0:1788204775.658047:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896576, portal 4 00000100:00000001:2.0:1788204775.658049:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204775.658051:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204775.658058:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204775.658065:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204775.658067:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204775.658070:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204775.658072:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205896576, offset 0 00000400:00000200:2.0:1788204775.658077:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204775.658081:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.658084:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204775.658088:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204775.658090:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204775.658092:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204775.658095:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204775.658097:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58860:58860:254:4294967295] 192.168.201.151@tcp LPNI seq info [58840:58840:6:4294967295] 00000400:00000200:2.0:1788204775.658101:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.658108:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204775.658112:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204775.658116:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4c00. 00000800:00000200:2.0:1788204775.658120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204775.658124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204775.658127:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204775.658131:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204775.658135:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204775.658137:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300902953 00000100:00000001:2.0:1788204775.658139:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.658141:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300902953 00000100:00000001:2.0:1788204775.658143:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.658146:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204775.658148:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.658150:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.658153:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.658157:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204775.658158:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204775.658160:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204775.658162:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204775.658163:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204775.658168:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204775.658176:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204775.658178:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.658181:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204775.658182:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204775.658185:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204775.658188:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205896704:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204775.658191:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204775.658194:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204775.658195:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204775.658197:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204775.658201:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c000. 02000000:00000001:2.0:1788204775.658203:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204775.658206:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204775.658209:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa630. 00000400:00000010:2.0:1788204775.658213:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85aa0. 00000100:00000200:2.0:1788204775.658217:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205896704, portal 4 00000100:00000001:2.0:1788204775.658219:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204775.658221:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204775.658226:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204775.658234:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204775.658236:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204775.658238:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204775.658241:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205896704, offset 0 00000400:00000200:2.0:1788204775.658246:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204775.658250:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.658253:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204775.658257:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204775.658260:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204775.658262:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204775.658265:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204775.658268:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58861:58861:253:4294967295] 192.168.201.151@tcp LPNI seq info [58841:58841:5:4294967295] 00000400:00000200:2.0:1788204775.658273:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204775.658280:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204775.658285:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204775.658288:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204775.658293:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204775.658296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204775.658299:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204775.658302:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204775.658305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204775.658307:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300902959 00000100:00000001:2.0:1788204775.658310:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.658312:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300902959 00000100:00000001:2.0:1788204775.658315:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300907010 00000100:00000001:2.0:1788204775.658317:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204775.658319:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.658322:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.658324:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.658327:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204775.658329:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204775.658331:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204775.658333:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204775.658335:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.658337:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204775.848450:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204775.848457:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204775.848462:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204775.848464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204775.848467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204775.848468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204775.848472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204775.848473:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204775.848547:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204775.848552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204775.848556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204776.168447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204776.168453:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204776.168454:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204776.168458:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204776.168461:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204776.168465:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204776.168467:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204776.168468:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204776.168470:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204776.616490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204776.616498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204776.616499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204776.616504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204776.616506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204776.616509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204776.616510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204776.616512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204776.616514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204776.680490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204776.680495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204776.680497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204776.680500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204776.680502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204776.680506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204776.680508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204776.680510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204776.680512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204776.872515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204776.872521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204776.872526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204776.872528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204776.872531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204776.872532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204776.872535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204776.872536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204776.872553:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204776.872555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204776.872558:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204777.192468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204777.192472:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204777.192474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.192477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204777.192481:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204777.192484:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204777.192487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204777.192490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204777.192492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204777.640468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204777.640474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204777.640477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204777.640481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204777.640485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204777.640488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204777.640490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204777.640492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204777.640495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204777.704485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204777.704493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204777.704495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204777.704499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204777.704503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204777.704506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204777.704509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204777.704512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204777.704515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204777.896461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204777.896466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204777.896471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204777.896472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204777.896474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204777.896475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204777.896477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204777.896478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204777.896535:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204777.896538:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204777.896542:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204778.216475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204778.216481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204778.216482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204778.216486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204778.216489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204778.216491:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204778.216493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204778.216495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204778.216497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204778.664443:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204778.664448:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204778.664449:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204778.664453:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204778.664455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204778.664458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204778.664459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204778.664461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204778.664463:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204778.728470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204778.728484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204778.728487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204778.728492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204778.728495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204778.728498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204778.728502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204778.728505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204778.728508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788204778.728681:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:2.0:1788204778.728717:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788204778.728726:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204778.728730:0:2336:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:2.0:1788204778.728735:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204778.728738:0:2336:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1788204778.728743:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788204778.728747:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000400:00000200:2.0:1788204778.728750:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204778.728754:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204778.728765:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:2.0:1788204778.728768:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:2.0:1788204778.728773:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204778.728783:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728810:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204778.728815:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205896320/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728822:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.728825:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204778.728834:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788204778.728836:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4400. 00000400:00000200:2.0:1788204778.728839:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204778.728843:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204778.728846:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85908 00000400:00000010:2.0:1788204778.728848:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85908. 00000100:00000001:2.0:1788204778.728850:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204778.728854:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728862:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204778.728865:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205896448/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728871:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.728873:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204778.728878:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788204778.728881:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4c00. 00000400:00000200:2.0:1788204778.728883:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204778.728886:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204778.728889:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204778.728891:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204778.728893:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204778.728897:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728905:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204778.728908:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205896576/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728914:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.728915:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204778.728920:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788204778.728922:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204778.728924:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204778.728928:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204778.728930:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204778.728932:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204778.728934:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204778.728938:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728945:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204778.728948:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205896704/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204778.728954:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.728955:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1788204778.920515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204778.920520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204778.920524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204778.920525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204778.920527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204778.920528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204778.920529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204778.920530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204778.920543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204778.920544:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204778.920546:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204779.240455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204779.240461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204779.240463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204779.240467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204779.240470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204779.240473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204779.240475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204779.240477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204779.240479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204779.688470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204779.688477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204779.688479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204779.688485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204779.688488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204779.688492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204779.688495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204779.688498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204779.688501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204779.752474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204779.752480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204779.752481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204779.752485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204779.752487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204779.752490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204779.752492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204779.752494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204779.752497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944448:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204779.944454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204779.944462:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204779.944464:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204779.944467:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944470:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204779.944472:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944474:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944476:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204779.944481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204779.944484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204779.944485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204779.944529:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204779.944532:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204779.944535:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204780.264467:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300907010, at 4300907520 00000100:00000040:1.0:1788204780.264467:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300907010, at 4300907520 00000100:00000001:2.0:1788204780.264468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204780.264473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204780.264476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204780.264479:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00000010:1.0:1788204780.264480:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00100000:2.0:1788204780.264481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204780.264486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204780.264490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204780.264492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204780.264495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204780.264498:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204780.712420:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204780.712426:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204780.712427:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204780.712431:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204780.712434:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204780.712447:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204780.712450:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204780.712452:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204780.712455:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204780.776474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204780.776482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204780.776484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204780.776489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204780.776493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204780.776494:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204780.776497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204780.776502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204780.776502:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204780.776507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204780.776507:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204780.776513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204780.776513:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204780.776520:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204780.776528:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6235 00000400:00000200:2.0:1788204780.776542:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204780.776551:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204780.776557:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204780.776562:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776565:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776568:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776571:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204780.776574:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204780.776577:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204780.776579:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204780.776582:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204780.776584:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204780.776586:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204780.776589:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776591:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776593:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776595:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204780.776597:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204780.776600:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776603:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204780.776605:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204780.776609:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204780.776611:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776614:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204780.776616:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204780.776623:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204780.776626:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204780.776630:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204780.776631:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776634:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204780.776637:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204780.776640:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204780.776646:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204780.776652:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204780.776660:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205896832/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204780.776669:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204780.776674:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204780.776678:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205896832/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204780.776700:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204780.776703:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204780.776706:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204780.776708:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204780.776709:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204780.776712:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204780.776714:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6235 00000400:00000200:2.0:1788204780.776717:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204780.776720:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204780.776722:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204780.776725:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776727:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776729:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776731:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204780.776733:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204780.776734:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204780.776736:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204780.776738:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204780.776740:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204780.776741:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204780.776744:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776746:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776748:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776758:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204780.776761:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204780.776763:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776765:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204780.776767:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204780.776771:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204780.776773:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776775:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204780.776779:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204780.776781:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776783:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204780.776786:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204780.776788:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204780.776790:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204780.776791:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776793:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204780.776794:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204780.776796:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204780.776799:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204780.776802:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204780.776806:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205896960/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204780.776812:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204780.776815:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204780.776819:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205896960/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204780.776826:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204780.776828:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204780.776831:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204780.776832:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204780.776833:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204780.776836:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204780.776838:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6235 00000400:00000200:2.0:1788204780.776841:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204780.776844:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204780.776846:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204780.776848:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776850:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776852:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776854:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204780.776856:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204780.776857:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204780.776858:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204780.776859:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204780.776861:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204780.776863:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204780.776865:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776867:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776868:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776870:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204780.776872:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204780.776874:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776877:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204780.776878:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204780.776881:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204780.776882:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776884:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204780.776886:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204780.776888:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776890:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204780.776893:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204780.776894:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204780.776897:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204780.776898:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.776899:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204780.776900:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204780.776901:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204780.776905:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788204780.776907:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204780.776911:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205897088/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204780.776918:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204780.776919:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204780.776923:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205897088/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204780.776929:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204780.776931:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204780.776944:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204780.776946:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204780.776947:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204780.776950:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204780.776952:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6235 00000400:00000200:2.0:1788204780.776956:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204780.776958:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204780.776960:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204780.776963:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776965:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776967:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776969:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204780.776971:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204780.776972:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204780.776973:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204780.776975:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204780.776977:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204780.776979:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204780.776982:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776984:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204780.776986:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776988:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204780.776991:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204780.776994:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204780.776996:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204780.776998:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204780.777003:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204780.777007:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.777011:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204780.777013:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204780.777018:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.777022:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204780.777032:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204780.777036:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204780.777040:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204780.777043:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.777048:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204780.777051:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204780.777055:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204780.777061:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204780.777065:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204780.777073:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205897216/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204780.777084:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204780.777087:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204780.777092:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205897216/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204780.777104:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204780.777109:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6245) 00000800:00000200:3.1:1788204780.777795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204780.777801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204780.778154:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204780.778156:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204780.778721:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204780.778723:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204780.779110:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204780.779113:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00010000:00000001:1.0:1788204780.968522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204780.968528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204780.968533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204780.968534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204780.968537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204780.968539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204780.968542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204780.968543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204780.968561:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204780.968563:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204780.968566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204781.288471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204781.288476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204781.288478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204781.288482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204781.288485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204781.288487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204781.288489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204781.288491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204781.288493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204781.736465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204781.736470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204781.736472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204781.736476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204781.736479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204781.736481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204781.736483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204781.736485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204781.736486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204781.800464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204781.800471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204781.800473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204781.800477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204781.800480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204781.800483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204781.800485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204781.800486:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204781.800488:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204781.992479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204781.992485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204781.992491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204781.992492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204781.992496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204781.992498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204781.992501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204781.992503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204781.992557:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204781.992560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204781.992562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204782.312450:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204782.312461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204782.312463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.312468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204782.312471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204782.312474:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204782.312476:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204782.312478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204782.312481:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204782.760488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204782.760494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204782.760497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204782.760503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204782.760507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204782.760510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204782.760513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204782.760515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204782.760518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204782.824463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204782.824470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204782.824472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204782.824478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204782.824483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204782.824487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204782.824492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204782.824496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204782.824500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204783.016568:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204783.016570:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204783.016573:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016578:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204783.016580:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016582:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016584:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204783.016590:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016593:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204783.016594:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204783.016596:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204783.016597:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016599:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204783.016600:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016601:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016603:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016610:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204783.016612:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204783.016613:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204783.016614:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016616:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204783.016618:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016619:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204783.016621:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204783.016641:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204783.016644:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204783.016646:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204783.337401:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204783.337409:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204783.337410:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204783.337416:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204783.337419:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204783.337424:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204783.337427:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204783.337429:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204783.337432:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204783.784465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204783.784475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204783.784477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204783.784482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204783.784486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204783.784490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204783.784493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204783.784495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204783.784498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204783.848438:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204783.848452:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204783.848459:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204783.848464:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204783.848470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204783.848474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204783.848478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204783.848481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204783.848484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204784.040471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204784.040477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204784.040480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204784.040481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204784.040483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204784.040484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204784.040486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204784.040488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204784.040548:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204784.040552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204784.040555:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204784.360506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204784.360514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204784.360515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204784.360520:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204784.360523:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204784.360527:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204784.360530:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204784.360533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204784.360535:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204784.808484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204784.808490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204784.808492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204784.808498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204784.808503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204784.808508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204784.808511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204784.808513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204784.808515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204784.872511:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204784.872523:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204784.872528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204784.872537:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204784.872542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204784.872547:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204784.872551:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204784.872556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204784.872559:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204785.064500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204785.064505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204785.064510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204785.064511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204785.064512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204785.064513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204785.064515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204785.064516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204785.064529:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204785.064531:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204785.064533:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204785.384475:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300912130, at 4300912640 00000100:00000040:0.0:1788204785.384477:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300912130, at 4300912640 00000100:00000001:2.0:1788204785.384478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204785.384483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204785.384488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204785.384491:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00100000:2.0:1788204785.384491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204785.384494:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000001:2.0:1788204785.384494:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204785.384505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204785.384509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204785.384512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204785.384515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204785.832452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204785.832458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204785.832460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204785.832465:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204785.832468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204785.832472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204785.832475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204785.832477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204785.832480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204785.896439:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204785.896449:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204785.896452:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204785.896456:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204785.896460:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204785.896468:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6240 00000400:00000200:2.0:1788204785.896476:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204785.896482:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204785.896487:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204785.896493:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896497:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896502:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896506:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204785.896511:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204785.896514:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204785.896519:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204785.896523:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204785.896525:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204785.896528:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204785.896536:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896538:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896541:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896543:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204785.896547:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204785.896551:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896553:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204785.896556:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204785.896559:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204785.896562:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896564:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204785.896566:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204785.896574:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204785.896578:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204785.896581:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204785.896582:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896585:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204785.896588:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204785.896591:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204785.896598:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788204785.896604:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204785.896612:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205897344/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204785.896621:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204785.896625:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204785.896629:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205897344/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204785.896650:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204785.896653:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204785.896656:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204785.896657:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204785.896659:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204785.896661:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204785.896664:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6240 00000400:00000200:2.0:1788204785.896667:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204785.896670:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204785.896672:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204785.896675:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896678:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896681:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896683:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204785.896685:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204785.896689:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204785.896691:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204785.896695:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204785.896699:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204785.896703:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204785.896708:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896712:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896716:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896720:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204785.896726:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204785.896731:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896736:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204785.896740:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204785.896746:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204785.896750:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896755:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204785.896759:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204785.896763:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896767:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204785.896771:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204785.896775:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204785.896779:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204785.896782:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896784:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204785.896786:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204785.896790:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204785.896797:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788204785.896803:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204785.896811:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205897472/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204785.896821:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204785.896824:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204785.896828:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205897472/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204785.896843:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204785.896847:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204785.896854:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204785.896857:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204785.896860:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204785.896866:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204785.896872:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6240 00000400:00000200:2.0:1788204785.896879:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204785.896884:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204785.896888:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204785.896894:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896898:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896901:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896903:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204785.896908:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204785.896911:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204785.896913:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204785.896915:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204785.896918:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204785.896923:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204785.896929:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896933:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.896938:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896943:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204785.896949:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204785.896955:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204785.896959:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204785.896964:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204785.896968:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204785.896972:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896974:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204785.896978:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204785.896983:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.896986:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204785.896992:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204785.896997:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204785.897001:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204785.897002:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.897005:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204785.897007:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204785.897010:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204785.897016:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204785.897021:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204785.897031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:2.0:1788204785.897033:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205897600/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204785.897038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788204785.897047:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204785.897050:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204785.897057:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205897600/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204785.897069:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204785.897074:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204785.897080:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204785.897082:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204785.897084:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204785.897088:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204785.897094:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6240 00000400:00000200:2.0:1788204785.897101:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204785.897117:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204785.897123:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204785.897129:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.897134:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.897138:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.897142:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204785.897146:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204785.897150:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204785.897152:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204785.897155:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204785.897159:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204785.897165:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204785.897171:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.897174:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204785.897177:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204785.897180:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204785.897185:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204785.897188:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204785.897190:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204785.897195:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204785.897198:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204785.897200:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.897202:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204785.897203:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204785.897205:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.897206:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204785.897210:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204785.897212:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204785.897215:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204785.897219:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.897221:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204785.897224:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204785.897235:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204785.897240:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204785.897244:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204785.897253:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205897728/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204785.897266:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204785.897270:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204785.897278:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205897728/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204785.897287:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204785.897289:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204785.897292:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204785.897295:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204785.897295:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6250) 00000100:00100000:1.0:1788204785.897300:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204785.897303:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204785.897306:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204785.897309:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204785.897311:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204785.897315:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204785.897621:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204785.897624:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204785.897898:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204785.897900:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204785.898167:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204785.898169:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00010000:00000001:1.0:1788204786.088467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204786.088472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204786.088476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204786.088477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204786.088480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204786.088481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204786.088483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204786.088484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204786.088527:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204786.088530:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204786.088532:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204786.856473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204786.856481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204786.856483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204786.856489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204786.856492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204786.856496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204786.856498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204786.856500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204786.856503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204786.920472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204786.920478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204786.920480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204786.920485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204786.920488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204786.920492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204786.920495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204786.920497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204786.920501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204787.112487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204787.112493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204787.112498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204787.112499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204787.112503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204787.112504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204787.112507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204787.112508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204787.112524:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204787.112526:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204787.112529:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204787.880437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204787.880443:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204787.880445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204787.880450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204787.880453:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204787.880456:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204787.880459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204787.880461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204787.880464:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204787.944467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204787.944474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204787.944476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204787.944481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204787.944484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204787.944488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204787.944496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204787.944497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204787.944499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204788.136469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204788.136474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204788.136478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204788.136479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204788.136482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204788.136483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204788.136486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204788.136486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204788.136542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204788.136545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204788.136548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204788.904479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204788.904486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204788.904488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204788.904493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204788.904496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204788.904500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204788.904502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204788.904505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204788.904508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204788.968452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204788.968460:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204788.968463:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204788.968468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204788.968472:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204788.968475:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204788.968484:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204788.968487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204788.968490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204789.160508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204789.160513:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204789.160515:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204789.160517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160519:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204789.160521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204789.160526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204789.160528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204789.160528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204789.160541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204789.160543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204789.160544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204789.928456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204789.928462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204789.928463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204789.928467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204789.928469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204789.928472:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204789.928473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204789.928475:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204789.928478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204789.992465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204789.992471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204789.992472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204789.992483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204789.992486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204789.992490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204789.992493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204789.992495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204789.992498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204790.184461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204790.184466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204790.184469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204790.184470:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204790.184473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204790.184474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204790.184476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204790.184477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204790.184531:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204790.184535:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204790.184538:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204790.504429:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300917249, at 4300917760 00000100:00000040:1.0:1788204790.504435:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300917249, at 4300917760 00000100:00000010:0.0:1788204790.504440:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00000010:1.0:1788204790.504447:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:1.0:1788204790.952440:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204790.952447:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204790.952449:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204790.952454:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204790.952456:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204790.952459:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204790.952461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204790.952463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204790.952465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204791.016475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204791.016480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204791.016483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204791.016492:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204791.016492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204791.016498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204791.016500:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204791.016505:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204791.016505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204791.016512:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204791.016518:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204791.016522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204791.016526:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6245 00000100:00000001:1.0:1788204791.016526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204791.016531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204791.016539:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204791.016546:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204791.016550:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204791.016553:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016556:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.016558:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016560:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204791.016564:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204791.016566:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204791.016569:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204791.016572:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204791.016574:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204791.016576:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204791.016579:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016581:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.016583:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016584:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204791.016588:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204791.016591:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016593:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204791.016596:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204791.016599:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204791.016601:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.016604:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204791.016606:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204791.016615:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204791.016619:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204791.016623:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204791.016624:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.016627:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204791.016629:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204791.016633:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204791.016641:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204791.016647:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204791.016656:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205897856/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204791.016666:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204791.016670:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204791.016674:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205897856/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204791.016692:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204791.016695:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.016698:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204791.016700:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204791.016701:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204791.016704:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204791.016707:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6245 00000400:00000200:0.0:1788204791.016712:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204791.016716:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204791.016718:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204791.016722:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016726:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.016731:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016735:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204791.016738:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204791.016741:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204791.016745:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204791.016749:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204791.016754:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204791.016759:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204791.016767:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016773:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.016779:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016783:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204791.016791:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204791.016797:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016802:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204791.016806:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204791.016813:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204791.016818:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.016821:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204791.016827:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204791.016832:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.016836:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204791.016841:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204791.016845:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204791.016848:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204791.016851:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.016855:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204791.016858:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204791.016861:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204791.016867:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204791.016873:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204791.016883:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205897984/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204791.016897:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204791.016902:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204791.016910:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205897984/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204791.016922:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204791.016928:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.016933:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204791.016937:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204791.016940:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204791.016945:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204791.016952:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6245 00000400:00000200:0.0:1788204791.016961:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204791.016969:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204791.016974:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204791.016982:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016988:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.016992:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.016995:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204791.016999:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204791.017002:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204791.017005:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204791.017009:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204791.017014:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204791.017018:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204791.017025:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017031:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.017034:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017039:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204791.017046:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204791.017053:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.1:1788204791.017058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204791.017059:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204791.017064:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788204791.017065:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 02000000:00000001:0.0:1788204791.017068:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204791.017074:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017079:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204791.017083:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204791.017087:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017091:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204791.017097:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204791.017102:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204791.017108:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204791.017110:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017114:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204791.017115:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204791.017119:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204791.017124:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204791.017129:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204791.017135:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205898112/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204791.017158:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204791.017160:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204791.017164:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205898112/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204791.017194:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204791.017199:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.017205:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204791.017207:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204791.017211:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204791.017218:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204791.017223:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6245 00000400:00000200:0.0:1788204791.017231:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204791.017235:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204791.017237:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204791.017240:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017243:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.017248:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017253:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204791.017257:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204791.017269:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204791.017274:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204791.017278:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204791.017281:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204791.017285:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204791.017291:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017296:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204791.017299:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017303:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204791.017308:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204791.017320:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204791.017328:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204791.017330:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204791.017335:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204791.017342:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017346:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204791.017349:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204791.017354:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017358:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204791.017404:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204791.017411:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204791.017415:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204791.017419:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204791.017422:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204791.017426:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204791.017429:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204791.017436:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204791.017443:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204791.017453:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205898240/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204791.017472:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204791.017477:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204791.017486:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205898240/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204791.017504:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204791.017509:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6255) 00000800:00000200:3.1:1788204791.017702:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204791.017716:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204791.017987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204791.017989:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204791.018261:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204791.018263:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00010000:00000001:1.0:1788204791.208514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204791.208520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204791.208524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204791.208526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204791.208528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204791.208530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204791.208532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204791.208533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204791.208549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204791.208551:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204791.208554:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204791.976450:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204791.976457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204791.976458:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204791.976462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204791.976465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204791.976468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204791.976470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204791.976472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204791.976473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204792.040476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204792.040483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204792.040485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204792.040491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204792.040496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204792.040500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204792.040503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204792.040506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204792.040508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204792.232475:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204792.232477:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204792.232481:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232486:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204792.232488:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232490:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232493:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204792.232498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204792.232504:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204792.232505:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204792.232507:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232513:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204792.232515:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232517:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232519:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204792.232523:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204792.232525:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204792.232526:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232528:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204792.232529:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232531:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204792.232533:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204792.232592:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204792.232596:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204792.232599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204793.000452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204793.000459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204793.000461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204793.000466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204793.000469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204793.000473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204793.000476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204793.000478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204793.000480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204793.064445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204793.064451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204793.064455:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204793.064459:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204793.064465:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204793.064470:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204793.064474:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204793.064479:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204793.064483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204793.256496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204793.256504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204793.256509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204793.256511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204793.256515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204793.256516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204793.256519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204793.256520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204793.256538:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204793.256541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204793.256544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204794.024466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204794.024473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204794.024476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204794.024481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204794.024484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204794.024488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204794.024489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204794.024491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204794.024493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204794.088465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204794.088470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204794.088474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204794.088480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204794.088486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204794.088491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204794.088498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204794.088500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204794.088502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204794.280467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204794.280473:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204794.280476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204794.280477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204794.280479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204794.280480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204794.280483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204794.280484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204794.280542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204794.280546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204794.280549:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204795.048422:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204795.048427:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204795.048428:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204795.048432:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204795.048435:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204795.048439:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204795.048442:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204795.048444:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204795.048447:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204795.112451:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204795.112460:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204795.112464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204795.112471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204795.112477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204795.112481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204795.112492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204795.112496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204795.112498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204795.304514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204795.304520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204795.304527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204795.304528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204795.304531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204795.304532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204795.304536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204795.304537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204795.304554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204795.304557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204795.304560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204795.624458:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300922369, at 4300922880 00000100:00000040:0.0:1788204795.624468:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300922369, at 4300922880 00000100:00000010:1.0:1788204795.624479:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000010:0.0:1788204795.624481:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000001:1.0:1788204796.072475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204796.072481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204796.072482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204796.072486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204796.072488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204796.072491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204796.072492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204796.072494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204796.072496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204796.136469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204796.136477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204796.136481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204796.136488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204796.136494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204796.136495:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204796.136499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204796.136502:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204796.136505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204796.136506:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204796.136509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204796.136511:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204796.136514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136517:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204796.136524:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6250 00000400:00000200:2.0:1788204796.136532:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204796.136537:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204796.136539:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204796.136542:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136544:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136546:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136547:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204796.136549:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204796.136551:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204796.136553:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204796.136555:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204796.136556:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204796.136557:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204796.136559:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136560:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136561:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136563:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204796.136565:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204796.136567:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136568:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204796.136569:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204796.136572:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204796.136574:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136575:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204796.136577:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204796.136583:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204796.136586:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204796.136589:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204796.136590:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136593:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204796.136594:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204796.136597:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204796.136602:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204796.136607:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204796.136614:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205898368/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204796.136620:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204796.136623:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204796.136627:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205898368/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204796.136642:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204796.136645:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204796.136648:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204796.136650:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204796.136652:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204796.136655:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204796.136658:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6250 00000400:00000200:2.0:1788204796.136664:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204796.136671:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204796.136676:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204796.136683:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136688:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136692:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136695:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204796.136699:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204796.136701:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204796.136703:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204796.136706:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204796.136709:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204796.136711:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204796.136716:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136720:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136722:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136724:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204796.136726:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204796.136728:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136731:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204796.136733:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204796.136739:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204796.136742:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136747:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204796.136752:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204796.136755:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136760:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204796.136766:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204796.136770:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204796.136775:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204796.136776:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136780:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204796.136784:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204796.136789:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204796.136796:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204796.136802:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204796.136811:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205898496/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204796.136826:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204796.136830:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204796.136839:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205898496/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204796.136865:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204796.136869:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204796.136872:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204796.136875:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204796.136878:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204796.136882:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204796.136887:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6250 00000400:00000200:2.0:1788204796.136892:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204796.136896:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204796.136899:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204796.136904:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136907:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136909:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136913:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204796.136916:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:3.1:1788204796.136916:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:2.0:1788204796.136918:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204796.136921:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:3.1:1788204796.136922:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000020:00000040:2.0:1788204796.136923:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204796.136927:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204796.136929:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204796.136933:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136936:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.136939:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136941:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204796.136943:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204796.136945:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204796.136948:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204796.136950:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204796.136953:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204796.136956:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136958:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204796.136961:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204796.136963:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136964:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204796.136967:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204796.136971:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204796.136974:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204796.136977:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.136979:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204796.136980:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204796.136982:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204796.136987:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204796.136991:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204796.136997:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205898624/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204796.137020:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204796.137023:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204796.137028:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205898624/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204796.137037:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204796.137040:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204796.137044:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204796.137045:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204796.137046:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204796.137064:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204796.137067:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6250 00000400:00000200:2.0:1788204796.137069:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204796.137071:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204796.137073:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204796.137075:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.137076:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.137077:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.137080:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204796.137081:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204796.137082:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204796.137083:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204796.137085:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204796.137090:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204796.137091:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204796.137094:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.137095:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204796.137098:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204796.137100:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204796.137102:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204796.137105:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204796.137108:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204796.137109:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204796.137113:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204796.137115:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.137119:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204796.137121:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204796.137123:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.137127:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204796.137130:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204796.137133:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204796.137136:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204796.137138:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.137140:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204796.137142:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204796.137144:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204796.137148:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204796.137154:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204796.137159:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205898752/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204796.137168:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204796.137170:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204796.137176:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205898752/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204796.137183:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204796.137187:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6261) 00000800:00000200:3.1:1788204796.137418:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204796.137419:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204796.137584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204796.137586:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204796.137765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204796.137767:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00010000:00000001:1.0:1788204796.328463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204796.328469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204796.328474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204796.328476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204796.328479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204796.328481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204796.328484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204796.328485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204796.328542:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204796.328545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204796.328548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204797.096490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204797.096497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204797.096499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204797.096504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204797.096507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204797.096511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204797.096513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204797.096515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204797.096518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204797.160470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204797.160478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204797.160482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204797.160487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204797.160491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204797.160495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204797.160499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204797.160502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204797.160506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204797.352497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204797.352504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204797.352509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204797.352510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204797.352513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204797.352515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204797.352518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204797.352519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204797.352536:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204797.352538:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204797.352541:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204798.120469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204798.120477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204798.120479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204798.120484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204798.120487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204798.120491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204798.120494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204798.120497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204798.120500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204798.184458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204798.184464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204798.184466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204798.184472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204798.184476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204798.184481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204798.184486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204798.184490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204798.184492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204798.376521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204798.376529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204798.376534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204798.376536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204798.376540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204798.376541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204798.376546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204798.376547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204798.376608:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204798.376615:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204798.376619:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204799.144461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204799.144467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204799.144468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204799.144472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204799.144475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204799.144478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204799.144480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204799.144482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204799.144484:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204799.208484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204799.208493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204799.208497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204799.208506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204799.208515:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204799.208525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204799.208531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204799.208533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204799.208536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204799.400571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204799.400578:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204799.400579:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204799.400582:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400584:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204799.400585:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400587:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400588:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204799.400591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204799.400593:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204799.400593:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204799.400612:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204799.400615:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204799.400622:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204800.168428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204800.168437:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204800.168439:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204800.168444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204800.168447:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204800.168450:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204800.168452:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204800.168454:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204800.168456:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204800.232490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204800.232499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204800.232501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204800.232508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204800.232513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204800.232517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204800.232527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204800.232528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204800.232530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204800.424463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204800.424470:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204800.424476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204800.424477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204800.424479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204800.424481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204800.424483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204800.424484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204800.424525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204800.424534:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204800.424538:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204800.744490:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300927489, at 4300928000 00000100:00000040:1.0:1788204800.744490:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300927489, at 4300928000 00000100:00000010:0.0:1788204800.744511:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000010:1.0:1788204800.744513:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000001:1.0:1788204801.192490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204801.192498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204801.192500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204801.192505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204801.192508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204801.192512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204801.192514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204801.192516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204801.192519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204801.256471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204801.256472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204801.256474:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204801.256482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204801.256482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204801.256484:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204801.256488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204801.256488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256489:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204801.256497:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204801.256497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204801.256497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204801.256504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204801.256504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204801.256505:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204801.256511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204801.256512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204801.256514:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6256 00000100:00000001:2.0:1788204801.256518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204801.256522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204801.256525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204801.256527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788204801.256528:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204801.256531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204801.256536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204801.256539:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204801.256544:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204801.256548:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256552:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256555:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256557:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204801.256561:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204801.256564:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204801.256567:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204801.256571:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204801.256572:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204801.256574:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204801.256577:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256578:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256579:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256580:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204801.256582:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204801.256585:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256586:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204801.256589:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204801.256593:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204801.256596:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256598:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204801.256601:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204801.256613:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204801.256617:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204801.256622:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204801.256624:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256627:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204801.256630:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204801.256635:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204801.256646:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204801.256653:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204801.256662:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205898880/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204801.256670:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204801.256674:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204801.256678:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205898880/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204801.256700:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204801.256703:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204801.256706:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204801.256708:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204801.256709:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204801.256711:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204801.256713:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6256 00000400:00000200:0.0:1788204801.256716:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204801.256718:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204801.256721:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204801.256725:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256727:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256730:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256733:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204801.256735:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204801.256736:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204801.256738:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204801.256741:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204801.256742:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204801.256746:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204801.256752:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256755:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256758:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256761:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204801.256764:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204801.256768:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256771:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204801.256776:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204801.256781:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204801.256789:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256792:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204801.256797:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204801.256800:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256803:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204801.256806:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204801.256811:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204801.256814:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204801.256816:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256819:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204801.256821:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204801.256823:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204801.256829:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204801.256832:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204801.256838:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205899008/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204801.256848:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204801.256851:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204801.256858:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205899008/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204801.256883:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204801.256888:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204801.256893:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204801.256894:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204801.256895:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204801.256899:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204801.256903:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6256 00000400:00000200:0.0:1788204801.256905:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204801.256909:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204801.256912:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204801.256916:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256919:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256924:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256927:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204801.256930:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204801.256931:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204801.256933:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204801.256936:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204801.256939:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204801.256944:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204801.256948:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256951:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.256952:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256959:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204801.256974:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204801.256976:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204801.256977:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204801.256978:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204801.256981:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204801.256982:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256983:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204801.256985:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204801.256987:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.256991:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204801.256995:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204801.256999:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204801.257001:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204801.257002:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.257006:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204801.257009:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204801.257011:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204801.257016:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204801.257021:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204801.257028:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205899136/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204801.257042:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204801.257045:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204801.257053:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205899136/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204801.257062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204801.257068:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204801.257069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00080000:0.0:1788204801.257073:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204801.257088:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204801.257092:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204801.257094:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204801.257098:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204801.257102:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6256 00000400:00000200:0.0:1788204801.257109:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204801.257113:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204801.257116:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204801.257120:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.257122:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.257126:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.257130:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204801.257134:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204801.257135:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204801.257138:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204801.257140:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204801.257144:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204801.257148:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204801.257150:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.257152:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204801.257155:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204801.257157:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204801.257160:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204801.257166:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204801.257171:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204801.257174:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204801.257177:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204801.257180:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.257183:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204801.257185:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204801.257188:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.257189:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204801.257192:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204801.257194:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204801.257199:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204801.257201:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204801.257204:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204801.257206:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204801.257208:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204801.257212:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204801.257217:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204801.257224:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204801.257233:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204801.257236:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204801.257242:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204801.257252:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204801.257257:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6266) 00000800:00000200:3.1:1788204801.257498:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204801.257500:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204801.257706:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204801.257708:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00000200:3.0:1788204801.257858:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205899264, offset 0 00000400:00000200:3.0:1788204801.257861:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204801.257863:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204801.257865:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204801.257866:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204801.257868:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204801.257870:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204801.257871:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204801.257873:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58881:58881:256:4294967295] 192.168.201.151@tcp LPNI seq info [58861:58861:8:4294967295] 00000400:00000200:3.0:1788204801.257876:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204801.257879:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204801.257881:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204801.257883:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204801.257885:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204801.257887:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204801.257889:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204801.257891:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788204801.257894:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204801.257895:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204801.257896:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204801.257898:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204801.257900:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204801.257901:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204801.257903:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204801.257904:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.257909:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204801.257911:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.257914:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204801.257915:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204801.257916:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204801.257918:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204801.257919:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204801.257921:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300928559 00000100:00000001:3.0:1788204801.257923:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300932609 00000100:00000001:3.0:1788204801.257924:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300928559 00000100:00000001:3.0:1788204801.257925:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300932609 00000100:00000001:3.0:1788204801.257926:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204801.257927:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204801.257928:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204801.257930:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204801.257935:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204801.257936:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204801.257937:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204801.257939:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204801.257941:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204801.257942:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204801.257945:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204801.257953:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.257959:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204801.257971:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.257973:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204801.257975:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204801.257979:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.257986:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204801.257989:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.257995:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204801.257997:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.257998:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204801.257999:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258000:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204801.258001:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204801.258003:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.258005:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204801.258006:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204801.258008:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.258013:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899264:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204801.258015:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258016:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204801.258018:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205899008/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.258023:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204801.258024:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204801.258026:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204801.258028:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204801.258030:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204801.258031:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204801.258033:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204801.258035:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788204801.258038:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204801.258039:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204801.258040:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204801.258043:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204801.258044:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258046:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204801.258049:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205899136/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.258061:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204801.258062:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204801.258063:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204801.258065:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204801.258066:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204801.258067:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204801.258069:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204801.258071:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788204801.258074:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204801.258075:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204801.258076:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204801.258078:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204801.258079:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258080:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204801.258084:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205899264/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204801.258089:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204801.258090:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204801.258091:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204801.258094:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204801.258095:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204801.258096:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204801.258098:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204801.258099:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204801.258101:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204801.258102:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204801.258103:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204801.258105:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204801.258106:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258108:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.258111:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204801.258112:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204801.258114:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204801.258115:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204801.448528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204801.448536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204801.448540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204801.448542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204801.448544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204801.448545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204801.448548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204801.448549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204801.448565:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204801.448567:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204801.448569:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204801.768459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204801.768469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204801.768472:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204801.768479:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204801.768484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204801.768488:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.768491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204801.768495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204801.768498:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204802.216423:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204802.216431:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204802.216433:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204802.216438:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204802.216441:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204802.216445:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204802.216447:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204802.216449:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204802.216452:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204802.280485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204802.280485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204802.280494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204802.280494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204802.280498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204802.280501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204802.280505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204802.280507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204802.280512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204802.280512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204802.280519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204802.280519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204802.280524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204802.280528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204802.280532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204802.280532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204802.280536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204802.280540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204802.472483:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204802.472486:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204802.472489:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472494:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204802.472496:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472498:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472501:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204802.472508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204802.472514:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204802.472515:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204802.472517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472519:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204802.472520:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472524:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204802.472529:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204802.472531:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204802.472532:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472534:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204802.472535:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472536:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204802.472538:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204802.472573:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204802.472577:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204802.472581:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204802.792487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204802.792495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204802.792497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204802.792506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204802.792510:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204802.792514:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204802.792516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204802.792519:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204802.792521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204803.240533:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204803.240544:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204803.240553:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204803.240560:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204803.240564:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204803.240570:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204803.240573:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204803.240576:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204803.240580:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204803.304454:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204803.304462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204803.304464:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204803.304468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204803.304470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204803.304474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204803.304475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204803.304479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204803.304480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204803.304484:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788204803.304485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204803.304487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204803.304490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204803.304491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204803.304497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204803.304500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204803.304502:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204803.304505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204803.496540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204803.496548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204803.496555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204803.496557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204803.496560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204803.496561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204803.496565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204803.496566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204803.496588:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204803.496591:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204803.496594:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204803.816499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204803.816508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204803.816511:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204803.816516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204803.816520:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204803.816525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204803.816528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204803.816531:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204803.816535:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204804.264498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204804.264508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204804.264510:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204804.264517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204804.264520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204804.264525:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204804.264527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204804.264529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204804.264532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204804.328506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204804.328506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204804.328517:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204804.328517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204804.328522:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204804.328522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204804.328530:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204804.328531:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204804.328536:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204804.328537:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204804.328544:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204804.328544:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204804.328550:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204804.328553:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204804.328557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204804.328559:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204804.328561:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204804.328564:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204804.520466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204804.520474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204804.520479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204804.520480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204804.520483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204804.520484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204804.520487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204804.520488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204804.520543:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204804.520555:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204804.520558:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204804.840499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204804.840510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204804.840512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204804.840519:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204804.840524:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204804.840528:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204804.840532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204804.840535:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204804.840538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204805.288498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204805.288508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204805.288510:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204805.288517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204805.288522:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204805.288527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204805.288530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204805.288533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204805.288536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204805.352488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204805.352488:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204805.352498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204805.352498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204805.352502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204805.352503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204805.352508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204805.352509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204805.352512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204805.352514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204805.352516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204805.352519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204805.352520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204805.352523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204805.352526:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204805.352530:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204805.352534:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204805.352537:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204805.544533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204805.544540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204805.544545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204805.544546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204805.544548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204805.544549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204805.544552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204805.544553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204805.544570:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204805.544572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204805.544574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204805.864493:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300932609, at 4300933120 00000100:00000040:1.0:1788204805.864493:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300932609, at 4300933120 00000100:00000001:3.0:1788204805.864493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204805.864511:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788204805.864515:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000001:3.0:1788204805.864516:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204805.864520:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00100000:3.0:1788204805.864525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204805.864531:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204805.864537:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204805.864541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204805.864545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204805.864549:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204806.312496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204806.312504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204806.312506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204806.312511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204806.312515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204806.312520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204806.312523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204806.312526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204806.312530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204806.376477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204806.376480:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204806.376480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204806.376487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204806.376487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204806.376488:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204806.376491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204806.376492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376493:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788204806.376497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204806.376499:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204806.376499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204806.376502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204806.376505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204806.376506:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204806.376507:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204806.376509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.376510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204806.376516:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6261 00000100:00000001:3.0:1788204806.376516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.376519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204806.376522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204806.376526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204806.376527:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204806.376529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204806.376537:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204806.376541:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204806.376546:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376550:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.376552:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204806.376563:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204806.376565:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204806.376568:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204806.376571:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204806.376574:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204806.376576:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204806.376579:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376581:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.376583:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376586:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204806.376589:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204806.376592:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376595:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204806.376598:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204806.376602:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204806.376605:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376607:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204806.376611:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204806.376624:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204806.376628:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204806.376632:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204806.376634:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376637:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204806.376641:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204806.376645:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204806.376657:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204806.376664:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204806.376677:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204806.376688:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204806.376693:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204806.376698:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204806.376722:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204806.376725:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204806.376728:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204806.376730:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204806.376731:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204806.376734:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204806.376736:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6261 00000400:00000200:0.0:1788204806.376739:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204806.376742:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204806.376744:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204806.376746:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376748:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.376750:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376751:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204806.376752:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204806.376753:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204806.376756:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204806.376756:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204806.376760:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204806.376761:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1788204806.376764:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204806.376768:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204806.376769:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204806.376771:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204806.376778:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.376781:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.376784:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204806.376784:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204806.376786:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376787:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204806.376790:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204806.376791:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788204806.376794:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204806.376795:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204806.376796:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204806.376799:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204806.376800:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204806.376804:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788204806.376807:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205899392:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204806.376808:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376811:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204806.376816:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204806.376816:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788204806.376820:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376821:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.376822:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1788204806.376824:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 02000000:00000001:3.0:1788204806.376826:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204806.376827:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204806.376832:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204806.376833:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.376833:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204806.376836:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204806.376837:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204806.376838:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204806.376840:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000010:0.0:1788204806.376842:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204806.376844:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.376846:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204806.376852:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204806.376855:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204806.376862:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1788204806.376863:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204806.376864:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204806.376883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000040:0.0:1788204806.376886:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204806.376893:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205899392, portal 25 00000100:00000001:0.0:1788204806.376894:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.376899:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788204806.376900:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204806.376901:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00080000:0.0:1788204806.376907:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204806.376910:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:3.0:1788204806.376910:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204806.376914:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204806.376921:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204806.376921:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204806.376926:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204806.376928:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6261 00000400:00000010:3.0:1788204806.376930:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204806.376935:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205899392, offset 0 00000400:00000200:0.0:1788204806.376939:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.376943:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204806.376945:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204806.376949:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204806.376952:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204806.376955:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204806.376957:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.376962:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204806.376963:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204806.376967:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204806.376971:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204806.376976:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788204806.376980:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788204806.376981:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788204806.376983:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204806.376986:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204806.376987:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:0.0:1788204806.376990:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204806.376995:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204806.376995:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58882:58882:256:4294967295] 192.168.201.151@tcp LPNI seq info [58862:58862:8:4294967295] 00000020:00000001:0.0:1788204806.376999:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204806.377005:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204806.377006:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377010:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.377012:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377015:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204806.377019:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204806.377024:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:0.0:1788204806.377026:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377031:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204806.377033:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204806.377037:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1788204806.377037:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204806.377041:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.377043:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:3.0:1788204806.377046:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000008:00100000:0.0:1788204806.377053:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204806.377057:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204806.377058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204806.377059:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204806.377065:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000800:00000200:3.0:1788204806.377066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000020:00000040:0.0:1788204806.377068:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204806.377072:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204806.377074:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.377075:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204806.377078:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788204806.377079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204806.377080:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204806.377085:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000800:00000200:3.1:1788204806.377085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1788204806.377089:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204806.377094:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.377097:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00100000:0.0:1788204806.377101:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204806.377106:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204806.377114:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204806.377116:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788204806.377119:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204806.377123:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204806.377127:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000040:0.0:1788204806.377128:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377133:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204806.377138:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204806.377141:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204806.377146:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204806.377149:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788204806.377153:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:3.0:1788204806.377154:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204806.377157:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204806.377161:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204806.377164:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.377166:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204806.377166:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788204806.377172:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6261 00000100:00000001:3.0:1788204806.377174:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204806.377179:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.377181:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204806.377185:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204806.377186:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300937729 00000100:00080000:0.0:1788204806.377190:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1788204806.377194:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 00000100:00000040:0.0:1788204806.377196:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377200:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204806.377201:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377204:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204806.377205:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788204806.377207:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204806.377210:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204806.377212:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204806.377214:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300937729 00000020:00000040:0.0:1788204806.377215:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204806.377220:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000010:3.0:1788204806.377221:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f40c0. 00000020:00000001:0.0:1788204806.377224:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204806.377227:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1788204806.377229:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204806.377231:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204806.377233:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204806.377234:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377238:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788204806.377239:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377244:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204806.377247:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00100000:3.0:1788204806.377250:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204806/real 1788204806] req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204806.377251:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204806.377257:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204806.377262:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1788204806.377265:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788204806.377266:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1788204806.377267:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204806.377270:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 02000000:00000001:0.0:1788204806.377271:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204806.377274:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000001:0.0:1788204806.377275:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:3.0:1788204806.377277:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000008:00100000:0.0:1788204806.377280:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788204806.377280:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204806.377284:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204806.377286:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.377292:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204806.377294:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788204806.377298:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204806.377299:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204806.377301:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788204806.377304:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204806.377304:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.377306:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204806.377311:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788204806.377311:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204806.377314:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204806.377320:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204806.377321:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204806.377323:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1788204806.377326:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204806.377328:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377335:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:0.0:1788204806.377336:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:3.0:1788204806.377338:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377341:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204806.377342:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377343:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377347:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204806.377349:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:0.0:1788204806.377353:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204806.377353:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204806.377360:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204806.377386:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204806.377393:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204806.377395:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204806.377400:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204806.377408:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204806.377410:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205899392:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204806.377420:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377424:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204806.377425:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204806.377429:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205899392/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204806.377432:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6271) 00000100:00000001:3.0:1788204806.377440:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204806.377443:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204806.377447:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204806.377451:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204806.377453:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204806.377454:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204806.377457:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204806.377459:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:3.0:1788204806.377463:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204806.377466:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204806.377468:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204806.377471:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204806.377478:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377480:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377487:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204806.377488:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377490:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204806.377491:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204806.377492:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204806.377495:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377500:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204806.377502:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.377503:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204806.377504:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204806.377506:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204806.377507:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899520:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204806.377510:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204806.377515:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204806.377516:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.377518:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204806.377520:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204806.377521:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204806.377524:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204806.377526:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204806.377528:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204806.377530:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205899520, portal 10 00000100:00000001:3.0:1788204806.377531:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204806.377532:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204806.377537:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377544:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204806.377546:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204806.377548:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204806.377550:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205899520, offset 0 00000400:00000200:3.0:1788204806.377554:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204806.377558:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.377562:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204806.377565:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204806.377568:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204806.377570:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204806.377573:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204806.377576:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58883:58883:256:4294967295] 192.168.201.151@tcp LPNI seq info [58863:58863:8:4294967295] 00000400:00000200:3.0:1788204806.377580:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.377586:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204806.377589:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.377591:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204806.377594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204806.377596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204806.377599:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204806.377616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204806.377621:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.377622:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204806.377625:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204806.377628:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204806.377632:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204806.377634:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204806.377636:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204806.377640:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377648:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.377650:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377656:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377657:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204806.377660:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204806.377663:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204806.377665:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204806.377667:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300933787 00000100:00000001:3.0:1788204806.377669:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.377670:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300933787 00000100:00000001:3.0:1788204806.377672:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.377674:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204806.377675:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377676:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204806.377679:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204806/real 1788204806] req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204806.377684:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204806.377685:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204806.377685:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204806.377687:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204806.377688:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204806.377689:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204806.377690:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204806.377695:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377699:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204806.377700:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377701:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.377702:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204806.377704:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377707:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204806.377709:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377713:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204806.377715:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377716:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.377717:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377718:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204806.377719:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204806.377721:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204806.377723:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204806.377725:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204806.377727:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204806.377732:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899520:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204806.377733:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204806.377735:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377738:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204806.377739:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.377740:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204806.377741:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204806.377743:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204806.377744:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899648:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204806.377746:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204806.377747:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204806.377748:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.377748:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204806.377751:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204806.377752:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204806.377754:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204806.377755:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204806.377757:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204806.377759:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205899648, portal 4 00000100:00000001:3.0:1788204806.377760:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204806.377761:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204806.377765:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377773:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204806.377774:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204806.377775:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204806.377777:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205899648, offset 0 00000400:00000200:3.0:1788204806.377779:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204806.377781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.377783:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204806.377785:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204806.377786:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204806.377788:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204806.377789:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204806.377792:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58884:58884:256:4294967295] 192.168.201.151@tcp LPNI seq info [58864:58864:8:4294967295] 00000400:00000200:3.0:1788204806.377794:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.377799:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204806.377803:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.377806:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204806.377809:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204806.377811:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204806.377815:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204806.377817:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204806.377821:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.377823:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204806.377826:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204806.377829:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204806.377832:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204806.377834:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204806.377836:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204806.377839:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377847:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.377850:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377862:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377863:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204806.377867:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204806.377870:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204806.377884:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204806.377886:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300933673 00000100:00000001:3.0:1788204806.377888:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.377891:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300933673 00000100:00000001:3.0:1788204806.377893:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.377895:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204806.377897:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.377899:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204806.377901:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204806/real 1788204806] req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204806.377907:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204806.377908:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204806.377909:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204806.377911:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204806.377912:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204806.377914:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204806.377917:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204806.377924:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377931:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204806.377932:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377934:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.377935:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204806.377938:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377945:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204806.377949:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377955:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204806.377958:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377959:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.377960:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.377962:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204806.377964:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204806.377967:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204806.377970:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204806.377972:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204806.377976:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204806.377984:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899648:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204806.377988:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204806.377992:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.377998:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204806.377999:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.378002:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204806.378003:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204806.378005:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204806.378008:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204806.378011:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204806.378014:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204806.378015:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204806.378017:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204806.378021:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204806.378023:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204806.378026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204806.378029:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204806.378032:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204806.378035:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205899776, portal 4 00000100:00000001:3.0:1788204806.378036:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204806.378037:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204806.378041:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378046:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204806.378047:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204806.378049:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204806.378050:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205899776, offset 0 00000400:00000200:3.0:1788204806.378053:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204806.378055:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.378057:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204806.378059:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204806.378060:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204806.378062:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204806.378064:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204806.378066:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58885:58885:256:4294967295] 192.168.201.151@tcp LPNI seq info [58865:58865:8:4294967295] 00000400:00000200:3.0:1788204806.378068:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204806.378072:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204806.378075:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.378077:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204806.378079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204806.378081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204806.378083:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204806.378085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204806.378087:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204806.378089:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204806.378091:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204806.378093:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204806.378095:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204806.378096:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204806.378098:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204806.378100:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378105:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.378107:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204831 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378110:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.378111:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204806.378113:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204806.378115:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204806.378117:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204806.378118:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300933679 00000100:00000001:3.0:1788204806.378119:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.378121:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300933679 00000100:00000001:3.0:1788204806.378122:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300937729 00000100:00000001:3.0:1788204806.378123:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204806.378124:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.378125:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204806.378127:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204806/real 1788204806] req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204806.378132:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204806.378133:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204806.378134:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204806.378135:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204806.378136:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204806.378138:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204806.378139:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204806.378145:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378148:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204806.378149:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378150:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.378150:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204806.378152:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378156:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204806.378158:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378161:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204806.378163:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378164:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204806.378165:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378166:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204806.378167:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204806.378169:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204806.378170:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204806.378171:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204806.378173:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204806.378177:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205899776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204806.378180:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378182:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.378185:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205899520/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378192:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204806.378193:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204806.378195:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204806.378197:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204806.378199:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204806.378200:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204806.378202:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204806.378203:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204806.378206:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204806.378207:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204806.378208:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204806.378211:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204806.378212:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378214:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.378218:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205899648/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378224:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204806.378225:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204806.378226:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204806.378228:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204806.378230:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204806.378231:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204806.378232:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204806.378233:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788204806.378235:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204806.378236:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204806.378237:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204806.378239:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204806.378240:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378241:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204806.378243:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205899776/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204831 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204806.378247:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204806.378248:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204806.378249:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204806.378250:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204806.378251:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204806.378252:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204806.378253:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204806.378254:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788204806.378255:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204806.378256:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204806.378256:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204806.378258:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204806.378258:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378259:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.378262:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204806.378262:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204806.378264:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204806.378265:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204806.568501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204806.568509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204806.568516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204806.568518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204806.568522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204806.568524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204806.568527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204806.568529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204806.568572:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204806.568577:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204806.568581:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204806.888443:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204806.888448:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204806.888449:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204806.888453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204806.888461:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204806.888464:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.888469:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204806.888471:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204806.888472:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204807.336481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204807.336489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204807.336490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204807.336495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204807.336499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204807.336503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204807.336505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204807.336508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204807.336510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204807.400484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204807.400484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204807.400493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204807.400493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204807.400496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204807.400497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204807.400502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204807.400503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204807.400507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204807.400508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204807.400511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204807.400512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204807.400514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204807.400518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204807.400521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204807.400523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204807.400527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204807.400529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204807.592562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204807.592578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204807.592589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204807.592590:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204807.592595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204807.592596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204807.592601:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204807.592602:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204807.592626:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204807.592629:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204807.592632:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204807.912737:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204807.912744:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204807.912747:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204807.912754:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204807.912759:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204807.912763:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204807.912766:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204807.912768:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204807.912771:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204808.360506:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204808.360518:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204808.360521:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204808.360528:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204808.360532:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204808.360537:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204808.360541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204808.360543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204808.360547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204808.424492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204808.424492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204808.424502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204808.424505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204808.424509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204808.424512:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204808.424512:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204808.424517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204808.424518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204808.424521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204808.424526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204808.424526:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204808.424532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204808.424536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204808.424540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204808.424540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204808.424544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204808.424548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204808.616492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204808.616501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204808.616508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204808.616509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204808.616512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204808.616513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204808.616515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204808.616516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204808.616562:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204808.616565:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204808.616568:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204808.936523:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204808.936530:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204808.936532:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204808.936538:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204808.936541:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204808.936545:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204808.936548:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204808.936550:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204808.936552:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204809.384500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204809.384509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204809.384511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204809.384516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204809.384520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204809.384524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204809.384526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204809.384529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204809.384536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204809.448464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204809.448464:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204809.448471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204809.448471:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204809.448475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204809.448475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204809.448481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204809.448482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204809.448488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204809.448488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204809.448495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204809.448495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204809.448502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204809.448506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204809.448510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204809.448511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204809.448515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204809.448519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204809.640500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204809.640508:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204809.640510:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204809.640513:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640516:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204809.640518:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640519:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640521:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204809.640524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204809.640526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204809.640527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204809.640541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204809.640543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204809.640545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204809.960483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204809.960492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204809.960495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204809.960501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204809.960505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204809.960510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204809.960513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204809.960515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204809.960519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204810.408464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204810.408474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204810.408477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204810.408483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204810.408487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204810.408491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204810.408493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204810.408495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204810.408498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204810.472434:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204810.472434:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204810.472446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204810.472446:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204810.472450:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204810.472450:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204810.472459:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204810.472460:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204810.472467:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204810.472467:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204810.472473:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204810.472474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204810.472478:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204810.472481:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204810.472485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204810.472487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204810.472491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204810.472494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204810.664504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204810.664511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204810.664515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204810.664516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204810.664519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204810.664520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204810.664523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204810.664524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204810.664571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204810.664574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204810.664577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204810.984480:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300937729, at 4300938240 00000100:00000040:1.0:1788204810.984480:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300937729, at 4300938240 00000100:00000001:2.0:1788204810.984480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204810.984489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204810.984493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204810.984497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204810.984499:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00000001:2.0:1788204810.984502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788204810.984503:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:2.0:1788204810.984508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204810.984513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204810.984516:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204810.984520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204811.432533:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204811.432545:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204811.432547:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204811.432555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204811.432559:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204811.432564:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204811.432567:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204811.432571:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204811.432574:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204811.496498:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204811.496499:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204811.496499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204811.496508:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204811.496509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204811.496511:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204811.496513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.496514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.496516:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204811.496522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204811.496522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204811.496523:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204811.496529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204811.496530:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204811.496535:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.496536:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204811.496537:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204811.496541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204811.496546:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204811.496550:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6266 00000100:00000001:1.0:1788204811.496552:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.496553:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204811.496557:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.496561:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204811.496572:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204811.496581:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204811.496585:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204811.496590:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496594:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.496597:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496600:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204811.496604:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204811.496606:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204811.496609:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204811.496614:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204811.496617:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204811.496619:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204811.496622:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496623:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.496625:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496627:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204811.496631:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204811.496635:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496637:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204811.496641:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204811.496644:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204811.496648:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.496651:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204811.496654:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204811.496665:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204811.496669:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204811.496674:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204811.496676:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.496680:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204811.496683:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204811.496688:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204811.496701:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788204811.496709:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204811.496720:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204811.496731:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204811.496738:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204811.496743:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204811.496767:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204811.496770:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204811.496773:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204811.496775:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204811.496777:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204811.496780:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204811.496797:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788204811.496799:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6266 00000100:00000001:3.0:1788204811.496802:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204811.496805:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1788204811.496806:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204811.496810:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204811.496815:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204811.496820:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204811.496821:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496827:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.496832:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496837:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204811.496842:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204811.496845:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204811.496847:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204811.496851:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204811.496852:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204811.496853:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204811.496856:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204811.496857:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204811.496862:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.496864:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.496865:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204811.496865:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204811.496869:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788204811.496869:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788204811.496871:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204811.496874:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204811.496875:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00100000:3.0:1788204811.496881:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205899904:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204811.496882:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204811.496886:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204811.496889:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204811.496893:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204811.496894:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204811.496897:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.496898:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.496900:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204811.496902:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788204811.496906:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204811.496909:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204811.496909:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 00000100:00000001:0.0:1788204811.496912:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.496915:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204811.496918:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204811.496922:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:3.0:1788204811.496925:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204811.496927:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204811.496930:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.496932:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788204811.496933:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 02000000:00000001:0.0:1788204811.496934:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204811.496937:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204811.496943:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 02000000:00000010:0.0:1788204811.496944:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204811.496951:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204811.496954:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205899904, portal 25 00000100:00100000:0.0:1788204811.496958:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204811.496958:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204811.496960:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204811.496969:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204811.496970:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204811.496974:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.496982:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204811.496983:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204811.496987:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204811.496991:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204811.496996:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205899904, offset 0 00000100:00000001:0.0:1788204811.497002:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.497005:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204811.497007:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204811.497013:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204811.497015:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204811.497017:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204811.497018:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204811.497025:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204811.497030:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6266 00000400:00000200:3.0:1788204811.497031:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204811.497038:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.497042:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204811.497045:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204811.497049:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204811.497052:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:0.0:1788204811.497056:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204811.497061:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788204811.497062:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.497068:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204811.497069:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497074:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204811.497079:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58886:58886:256:4294967295] 192.168.201.151@tcp LPNI seq info [58866:58866:8:4294967295] 00000020:00000001:0.0:1788204811.497080:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204811.497083:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204811.497086:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204811.497089:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204811.497091:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204811.497095:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204811.497096:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204811.497102:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497105:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204811.497108:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497112:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204811.497117:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204811.497118:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:0.0:1788204811.497124:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497128:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204811.497132:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1788204811.497133:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204811.497137:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204811.497141:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204811.497141:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:0.0:1788204811.497144:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204811.497148:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204811.497153:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204811.497153:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204811.497157:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204811.497159:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00000010:0.0:1788204811.497164:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000800:00000200:3.1:1788204811.497166:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204811.497169:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.1:1788204811.497170:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 02000000:00000001:0.0:1788204811.497176:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:3.0:1788204811.497179:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204811.497180:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.497184:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:3.0:1788204811.497185:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 02000000:00000001:0.0:1788204811.497186:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204811.497190:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.497194:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:0.0:1788204811.497195:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204811.497201:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.497204:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1788204811.497210:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204811.497213:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204811.497218:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:0.0:1788204811.497223:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204811.497225:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204811.497227:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204811.497234:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204811.497235:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497254:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204811.497256:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.497260:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204811.497261:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204811.497265:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204811.497268:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204811.497271:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204811.497276:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.497277:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204811.497280:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788204811.497283:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6266 00000100:00000001:3.0:1788204811.497288:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204811.497289:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.497294:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204811.497297:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204811.497299:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300942849 00000100:00080000:0.0:1788204811.497302:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1788204811.497306:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f40c0. 00000100:00000001:3.0:1788204811.497314:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788204811.497319:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204811.497322:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300942849 00000100:00000001:0.0:1788204811.497324:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:3.0:1788204811.497326:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 00000100:00000040:0.0:1788204811.497329:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204811.497331:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204811.497335:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204811.497336:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204811.497341:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204811.497342:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204811.497343:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:0.0:1788204811.497345:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00100000:3.0:1788204811.497349:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204811/real 1788204811] req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204811.497350:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204811.497354:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204811.497360:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000001:3.0:1788204811.497362:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204811.497380:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204811.497381:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204811.497383:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204811.497384:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204811.497387:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204811.497391:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204811.497395:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204811.497404:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497410:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788204811.497410:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204811.497417:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204811.497422:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204811.497424:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1788204811.497426:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204811.497428:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204811.497432:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204811.497432:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.497434:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204811.497439:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204811.497442:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204811.497443:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204811.497447:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204811.497452:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.497454:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204811.497456:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1788204811.497457:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204811.497462:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204811.497465:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.497466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788204811.497471:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204811.497476:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204811.497479:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788204811.497481:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204811.497484:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497485:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204811.497488:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204811.497490:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788204811.497491:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204811.497493:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204811.497494:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497496:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204811.497499:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:0.0:1788204811.497502:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00080000:3.0:1788204811.497502:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204811.497508:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788204811.497509:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497515:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788204811.497519:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204811.497525:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204811.497537:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204811.497542:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204811.497547:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205899904:192.168.201.151@tcp:250:kworker.0 00000100:00000040:0.0:1788204811.497550:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497555:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497561:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204811.497565:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.497567:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205899904/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204811.497572:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6276) 00000100:00000001:3.0:1788204811.497579:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204811.497582:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204811.497585:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204811.497590:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204811.497592:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204811.497594:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204811.497597:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204811.497600:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788204811.497613:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204811.497616:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204811.497618:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204811.497621:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204811.497623:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497625:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497634:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204811.497635:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497638:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204811.497639:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204811.497640:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204811.497645:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497651:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204811.497652:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.497655:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204811.497657:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204811.497659:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204811.497662:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900032:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204811.497665:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204811.497668:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204811.497669:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.497671:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204811.497674:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204811.497676:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.497680:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204811.497682:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204811.497686:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204811.497689:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900032, portal 10 00000100:00000001:3.0:1788204811.497691:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204811.497692:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204811.497699:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497705:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204811.497708:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204811.497710:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204811.497713:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205900032, offset 0 00000400:00000200:3.0:1788204811.497717:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204811.497720:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.497724:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204811.497727:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204811.497730:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204811.497732:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204811.497734:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204811.497737:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58887:58887:256:4294967295] 192.168.201.151@tcp LPNI seq info [58867:58867:8:4294967295] 00000400:00000200:3.0:1788204811.497741:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.497748:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204811.497752:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.497756:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204811.497760:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204811.497763:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204811.497768:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204811.497770:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204811.497774:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.497776:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204811.497778:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204811.497782:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204811.497798:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204811.497800:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204811.497803:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204811.497807:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497814:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.497817:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497822:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497824:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204811.497827:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.497831:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204811.497834:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204811.497836:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300938907 00000100:00000001:3.0:1788204811.497839:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.497841:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300938907 00000100:00000001:3.0:1788204811.497843:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.497846:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204811.497848:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.497849:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204811.497853:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204811/real 1788204811] req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204811.497868:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204811.497869:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204811.497871:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204811.497874:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204811.497875:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204811.497877:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204811.497881:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204811.497889:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497895:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204811.497897:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497898:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.497899:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204811.497903:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497910:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204811.497913:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497920:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204811.497922:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497924:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.497925:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.497927:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204811.497929:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204811.497932:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204811.497936:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204811.497938:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204811.497943:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204811.497951:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900032:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204811.497955:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204811.497958:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.497964:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204811.497966:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.497968:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204811.497969:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204811.497971:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204811.497974:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900160:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204811.497977:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204811.497978:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204811.497980:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.497981:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204811.497985:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204811.497987:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.497990:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204811.497992:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204811.497995:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204811.497997:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900160, portal 4 00000100:00000001:3.0:1788204811.497999:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204811.498000:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204811.498006:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498012:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204811.498014:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204811.498016:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204811.498019:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205900160, offset 0 00000400:00000200:3.0:1788204811.498023:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204811.498026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.498030:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204811.498032:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204811.498035:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204811.498037:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204811.498039:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204811.498042:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58888:58888:256:4294967295] 192.168.201.151@tcp LPNI seq info [58868:58868:8:4294967295] 00000400:00000200:3.0:1788204811.498046:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.498052:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204811.498057:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.498060:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204811.498064:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204811.498066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204811.498070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204811.498073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204811.498076:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.498078:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204811.498080:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204811.498083:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204811.498086:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204811.498088:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204811.498090:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204811.498094:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498101:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.498104:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498110:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.498112:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204811.498115:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.498118:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204811.498121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204811.498123:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300938793 00000100:00000001:3.0:1788204811.498126:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.498128:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300938793 00000100:00000001:3.0:1788204811.498130:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.498132:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204811.498134:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.498136:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204811.498140:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204811/real 1788204811] req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204811.498147:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204811.498149:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204811.498150:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204811.498153:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204811.498155:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204811.498157:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204811.498160:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204811.498168:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498175:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204811.498176:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498178:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.498179:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204811.498183:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498190:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204811.498193:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498200:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204811.498203:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498204:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.498205:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498207:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204811.498209:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204811.498212:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204811.498215:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204811.498217:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204811.498221:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204811.498229:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900160:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204811.498233:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204811.498236:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498242:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204811.498243:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.498246:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204811.498247:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204811.498250:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204811.498252:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900288:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204811.498255:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204811.498257:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204811.498259:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204811.498261:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204811.498264:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204811.498266:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204811.498269:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204811.498271:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204811.498274:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204811.498277:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900288, portal 4 00000100:00000001:3.0:1788204811.498279:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204811.498280:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204811.498287:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498293:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204811.498295:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204811.498297:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204811.498299:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205900288, offset 0 00000400:00000200:3.0:1788204811.498303:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204811.498307:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.498310:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204811.498313:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204811.498315:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204811.498317:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204811.498320:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204811.498322:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58889:58889:256:4294967295] 192.168.201.151@tcp LPNI seq info [58869:58869:8:4294967295] 00000400:00000200:3.0:1788204811.498326:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204811.498332:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204811.498337:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.498340:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204811.498344:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204811.498347:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204811.498351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204811.498353:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204811.498356:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204811.498358:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204811.498361:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204811.498384:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204811.498387:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204811.498389:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204811.498392:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204811.498395:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498409:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.498412:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498419:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.498420:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204811.498423:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204811.498427:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204811.498430:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204811.498432:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300938800 00000100:00000001:3.0:1788204811.498435:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.498437:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300938800 00000100:00000001:3.0:1788204811.498439:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300942849 00000100:00000001:3.0:1788204811.498442:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204811.498444:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.498445:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204811.498449:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204811/real 1788204811] req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204811.498457:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204811.498459:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204811.498460:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204811.498463:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204811.498464:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204811.498467:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204811.498470:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204811.498477:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498484:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204811.498485:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498487:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.498488:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204811.498492:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498498:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204811.498502:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498509:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204811.498511:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498513:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204811.498514:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498516:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204811.498518:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204811.498521:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204811.498524:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204811.498526:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204811.498530:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204811.498539:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900288:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204811.498543:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498545:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.498548:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205900032/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498554:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204811.498556:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204811.498558:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204811.498561:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204811.498562:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204811.498564:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204811.498566:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204811.498568:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788204811.498571:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204811.498572:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204811.498573:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204811.498576:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204811.498577:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498579:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.498582:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205900160/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498589:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204811.498590:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204811.498591:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204811.498593:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204811.498595:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204811.498596:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204811.498598:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204811.498599:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788204811.498601:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204811.498602:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204811.498604:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204811.498606:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204811.498608:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498609:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204811.498613:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205900288/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204811.498619:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204811.498620:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204811.498621:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204811.498624:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204811.498625:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204811.498626:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204811.498628:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204811.498629:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:3.0:1788204811.498633:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204811.498634:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204811.498635:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204811.498638:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204811.498639:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498641:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204811.498645:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204811.498646:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204811.498648:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204811.498650:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204811.688513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204811.688521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204811.688526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204811.688527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204811.688529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204811.688530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204811.688533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204811.688533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204811.688547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204811.688549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204811.688552:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204812.008437:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204812.008444:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204812.008446:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204812.008452:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204812.008454:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204812.008458:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204812.008461:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204812.008463:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204812.008465:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204812.456425:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204812.456433:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204812.456435:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204812.456441:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204812.456452:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204812.456456:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204812.456459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204812.456462:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204812.456466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204812.520441:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204812.520450:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204812.520453:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204812.520458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204812.520461:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204812.520463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204812.520466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204812.520468:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204812.520471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204812.520475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204812.520475:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204812.520480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204812.520481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204812.520486:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204812.520489:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204812.520492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204812.520496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204812.520499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204812.712483:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204812.712486:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204812.712489:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712494:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204812.712496:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712498:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712500:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204812.712505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204812.712509:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204812.712510:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204812.712511:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712512:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204812.712512:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712513:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712514:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204812.712518:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204812.712519:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204812.712520:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712520:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204812.712521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204812.712523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204812.712569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204812.712574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204812.712576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204813.032488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204813.032496:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204813.032497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204813.032502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204813.032505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204813.032509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204813.032511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204813.032513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204813.032516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204813.480481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204813.480491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204813.480494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204813.480500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204813.480505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204813.480510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204813.480513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204813.480517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204813.480520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204813.544479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204813.544479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204813.544487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204813.544488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204813.544490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204813.544491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204813.544500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204813.544500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204813.544507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204813.544507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204813.544513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204813.544514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204813.544519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204813.544523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204813.544527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204813.544527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204813.544531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204813.544535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204813.736535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204813.736541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204813.736546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204813.736547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204813.736549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204813.736550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204813.736552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204813.736553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204813.736569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204813.736571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204813.736573:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204814.056477:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204814.056487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204814.056490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204814.056497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204814.056501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204814.056507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204814.056510:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204814.056513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204814.056517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204814.504497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204814.504506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204814.504508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204814.504512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204814.504515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204814.504519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204814.504522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204814.504524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204814.504526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204814.568457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204814.568457:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204814.568465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204814.568466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204814.568467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204814.568470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204814.568474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204814.568476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204814.568478:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204814.568480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204814.568482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204814.568485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204814.568486:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204814.568492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204814.568497:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204814.568498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204814.568502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204814.568505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204814.760480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204814.760494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204814.760501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204814.760503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204814.760507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204814.760509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204814.760513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204814.760515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204814.760589:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204814.760595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204814.760598:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204815.080538:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204815.080549:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204815.080552:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204815.080560:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204815.080565:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204815.080571:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204815.080575:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204815.080579:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204815.080583:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204815.528467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204815.528473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204815.528474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204815.528478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204815.528481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204815.528484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204815.528486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204815.528488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204815.528490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204815.592519:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204815.592519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204815.592528:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204815.592529:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204815.592532:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204815.592534:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204815.592540:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204815.592542:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204815.592545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204815.592551:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204815.592552:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204815.592557:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204815.592560:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204815.592561:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204815.592572:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204815.592573:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204815.592578:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204815.592582:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204815.784576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204815.784582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204815.784588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204815.784589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204815.784592:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204815.784593:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204815.784596:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204815.784597:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204815.784617:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204815.784620:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204815.784622:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204816.104489:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300942849, at 4300943360 00000100:00000001:2.0:1788204816.104491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204816.104496:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300942849, at 4300943360 00000100:00000001:2.0:1788204816.104499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204816.104504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204816.104512:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00100000:2.0:1788204816.104512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204816.104516:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:2.0:1788204816.104519:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204816.104526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204816.104531:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204816.104534:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204816.104538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.552475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204816.552483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204816.552484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204816.552488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204816.552491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204816.552493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204816.552495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204816.552496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204816.552498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204816.616497:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204816.616506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204816.616506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204816.616513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204816.616514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204816.616518:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.616525:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204816.616526:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1788204816.616527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204816.616530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204816.616536:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204816.616537:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204816.616538:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204816.616542:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204816.616545:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204816.616546:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204816.616546:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204816.616551:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.616554:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616555:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204816.616558:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204816.616565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204816.616569:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204816.616576:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6271 00000400:00000200:1.0:1788204816.616586:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204816.616594:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204816.616598:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204816.616605:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616610:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204816.616613:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616616:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204816.616620:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204816.616622:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204816.616624:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204816.616627:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204816.616630:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204816.616632:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204816.616642:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616644:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204816.616648:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616650:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204816.616653:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204816.616656:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616658:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204816.616661:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204816.616664:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204816.616667:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616669:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204816.616671:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204816.616680:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788204816.616690:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204816.616770:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204816.616772:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616775:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204816.616777:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204816.616780:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204816.616786:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:1.0:1788204816.616792:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204816.616799:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204816.616806:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204816.616809:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204816.616812:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204816.616828:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204816.616831:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204816.616834:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204816.616835:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204816.616836:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204816.616838:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204816.616840:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6271 00000400:00000200:1.0:1788204816.616844:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204816.616847:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204816.616850:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204816.616853:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788204816.616854:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204816.616858:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204816.616860:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204816.616861:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204816.616870:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204816.616875:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204816.616880:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204816.616890:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204816.616892:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204816.616895:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204816.616896:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 02000000:00000001:3.0:1788204816.616900:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1788204816.616901:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204816.616907:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:3.0:1788204816.616909:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788204816.616910:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204816.616915:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204816.616919:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204816.616922:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616926:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788204816.616931:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205900416:192.168.201.151@tcp:250:kworker.0 00000100:00000040:1.0:1788204816.616932:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.616937:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204816.616943:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204816.616949:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204816.616949:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788204816.616955:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204816.616956:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204816.616958:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204816.616960:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204816.616964:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.616964:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616973:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204816.616974:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1788204816.616977:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204816.616979:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616981:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204816.616983:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204816.616984:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204816.616986:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204816.616987:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.616988:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204816.616989:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204816.616991:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204816.616993:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000010:1.0:1788204816.616995:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:3.0:1788204816.616997:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617000:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617007:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:1.0:1788204816.617009:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204816.617013:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204816.617021:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:1.0:1788204816.617030:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204816.617032:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204816.617034:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900416, portal 25 00000100:00000040:1.0:1788204816.617036:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617040:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204816.617045:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000001:1.0:1788204816.617058:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204816.617061:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204816.617062:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204816.617079:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204816.617084:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204816.617091:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788204816.617093:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788204816.617098:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617106:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788204816.617107:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00080000:1.0:1788204816.617111:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6271 00000400:00000200:1.0:1788204816.617117:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:3.0:1788204816.617117:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205900416, offset 0 00000400:00000200:1.0:1788204816.617124:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204816.617129:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204816.617132:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617133:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204816.617139:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617140:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204816.617142:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204816.617143:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204816.617143:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204816.617147:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204816.617149:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204816.617151:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000001:1.0:1788204816.617152:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204816.617161:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617162:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204816.617164:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617166:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204816.617167:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204816.617169:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617171:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617174:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788204816.617175:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204816.617180:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204816.617182:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617183:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204816.617187:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204816.617192:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00000001:1.0:1788204816.617193:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617198:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204816.617206:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000020:00000040:1.0:1788204816.617209:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788204816.617210:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1788204816.617217:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204816.617218:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617220:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788204816.617225:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204816.617227:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204816.617231:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1788204816.617233:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617236:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000010:1.0:1788204816.617238:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000400:00000200:3.0:1788204816.617246:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58890:58890:256:4294967295] 192.168.201.151@tcp LPNI seq info [58870:58870:8:4294967295] 00000100:00000001:1.0:1788204816.617247:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204816.617259:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204816.617260:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204816.617272:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204816.617274:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617287:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:1.0:1788204816.617290:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204816.617313:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204816.617315:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1788204816.617324:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788204816.617325:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204816.617327:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204816.617330:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204816.617334:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204816.617336:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6271 00000400:00000200:1.0:1788204816.617339:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000010:3.0:1788204816.617339:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000400:00000200:1.0:1788204816.617349:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204816.617351:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204816.617354:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617361:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204816.617400:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617402:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788204816.617404:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:1.0:1788204816.617405:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204816.617408:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204816.617412:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:3.0:1788204816.617413:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000020:00000040:1.0:1788204816.617418:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204816.617422:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788204816.617427:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:1.0:1788204816.617428:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788204816.617440:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1788204816.617442:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204816.617445:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788204816.617451:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788204816.617454:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000010:3.0:1788204816.617456:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000001:1.0:1788204816.617458:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204816.617461:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204816.617465:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204816.617465:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788204816.617468:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204816.617473:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204816.617476:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204816.617481:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617482:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788204816.617484:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204816.617487:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204816.617489:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617490:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204816.617493:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000400:00000200:3.0:1788204816.617493:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000020:00000040:1.0:1788204816.617502:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000010:3.0:1788204816.617502:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 02000000:00000001:1.0:1788204816.617512:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204816.617513:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1788204816.617514:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617518:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1788204816.617520:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204816.617524:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204816.617532:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204816.617545:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:3.0:1788204816.617550:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788204816.617552:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204816.617555:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204816.617566:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204816.617567:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617578:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204816.617585:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204816.617587:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204816.617588:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617594:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204816.617598:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300947970 00000100:00000040:1.0:1788204816.617599:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204816.617604:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4000. 00000100:00000001:1.0:1788204816.617616:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617616:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204816.617627:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300947970 00000100:00000040:1.0:1788204816.617628:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6281) 00000100:00000010:3.0:1788204816.617632:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4de0. 00000100:00000001:3.0:1788204816.617636:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204816.617641:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204816.617644:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617646:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204816.617651:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204816/real 1788204816] req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204816.617660:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204816.617662:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204816.617664:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204816.617667:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204816.617669:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204816.617672:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204816.617675:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204816.617683:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617690:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204816.617692:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617707:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.617710:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204816.617715:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617722:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204816.617726:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617733:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204816.617737:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617738:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.617740:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617743:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204816.617745:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204816.617749:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204816.617753:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204816.617756:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204816.617760:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204816.617767:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205900416:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204816.617770:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617772:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.617774:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205900416/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617783:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204816.617786:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204816.617788:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204816.617791:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204816.617793:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204816.617795:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204816.617796:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204816.617798:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:3.0:1788204816.617802:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204816.617803:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204816.617805:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:3.0:1788204816.617807:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204816.617808:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617810:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.617816:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204816.617817:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617819:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204816.617820:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204816.617821:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204816.617824:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617829:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204816.617830:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.617831:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204816.617833:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204816.617834:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204816.617836:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900544:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204816.617838:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204816.617840:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204816.617840:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.617841:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204816.617844:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204816.617845:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.617848:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204816.617849:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204816.617852:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204816.617854:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900544, portal 10 00000100:00000001:3.0:1788204816.617855:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204816.617857:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:3.0:1788204816.617861:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617866:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204816.617868:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204816.617869:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204816.617871:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205900544, offset 0 00000400:00000200:3.0:1788204816.617873:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204816.617876:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.617879:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204816.617881:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204816.617882:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204816.617884:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204816.617886:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204816.617888:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58891:58891:256:4294967295] 192.168.201.151@tcp LPNI seq info [58871:58871:8:4294967295] 00000400:00000200:3.0:1788204816.617890:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.617895:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204816.617899:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.617903:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204816.617906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204816.617909:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204816.617914:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204816.617916:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204816.617920:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.617922:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204816.617925:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204816.617928:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204816.617932:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204816.617934:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204816.617937:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204816.617941:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617948:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.617951:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.617964:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617965:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204816.617969:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204816.617972:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204816.617976:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204816.617978:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300944027 00000100:00000001:3.0:1788204816.617980:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.617982:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300944027 00000100:00000001:3.0:1788204816.617984:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.617985:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204816.617987:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.617988:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204816.617990:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204816/real 1788204816] req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204816.617995:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204816.617996:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204816.617997:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204816.617998:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204816.617999:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204816.618001:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204816.618003:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204816.618008:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618011:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204816.618012:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618013:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618014:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204816.618016:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618020:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204816.618022:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618026:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204816.618028:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618028:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618029:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618030:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204816.618032:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204816.618033:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204816.618035:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204816.618036:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204816.618038:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204816.618042:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900544:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204816.618045:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204816.618047:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618052:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204816.618052:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.618054:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204816.618055:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204816.618056:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204816.618057:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900672:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204816.618059:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204816.618061:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204816.618063:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.618064:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204816.618066:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204816.618068:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.618070:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204816.618071:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204816.618074:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204816.618076:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900672, portal 4 00000100:00000001:3.0:1788204816.618077:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204816.618078:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:3.0:1788204816.618082:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618086:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204816.618087:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204816.618089:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204816.618090:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205900672, offset 0 00000400:00000200:3.0:1788204816.618092:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204816.618094:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.618096:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204816.618098:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204816.618099:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204816.618100:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204816.618102:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204816.618105:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58892:58892:256:4294967295] 192.168.201.151@tcp LPNI seq info [58872:58872:8:4294967295] 00000400:00000200:3.0:1788204816.618109:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.618115:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204816.618119:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.618122:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204816.618126:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204816.618128:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204816.618133:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204816.618135:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204816.618139:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.618141:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204816.618144:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204816.618148:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204816.618151:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204816.618153:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204816.618155:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204816.618158:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618166:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.618169:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618176:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.618177:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204816.618180:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204816.618184:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204816.618187:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204816.618189:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300943913 00000100:00000001:3.0:1788204816.618192:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.618194:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300943913 00000100:00000001:3.0:1788204816.618196:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.618198:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204816.618200:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.618202:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204816.618205:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204816/real 1788204816] req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204816.618210:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204816.618211:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204816.618212:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204816.618214:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204816.618215:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204816.618217:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204816.618219:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204816.618223:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618227:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204816.618228:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618229:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618229:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204816.618231:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618235:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204816.618237:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618241:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204816.618242:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618243:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618244:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618245:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204816.618247:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204816.618248:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204816.618250:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204816.618251:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204816.618253:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204816.618258:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900672:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204816.618260:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204816.618262:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618267:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204816.618268:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.618270:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204816.618271:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204816.618272:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204816.618274:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900800:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204816.618275:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204816.618277:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204816.618278:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204816.618279:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204816.618281:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204816.618282:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204816.618284:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204816.618285:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204816.618287:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204816.618289:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900800, portal 4 00000100:00000001:3.0:1788204816.618291:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204816.618292:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:3.0:1788204816.618298:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618304:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204816.618306:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204816.618309:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204816.618311:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205900800, offset 0 00000400:00000200:3.0:1788204816.618315:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204816.618319:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.618322:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204816.618325:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204816.618327:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204816.618329:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204816.618332:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204816.618336:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58893:58893:256:4294967295] 192.168.201.151@tcp LPNI seq info [58873:58873:8:4294967295] 00000400:00000200:3.0:1788204816.618340:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204816.618346:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204816.618350:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.618353:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204816.618357:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204816.618360:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204816.618382:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204816.618384:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204816.618386:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204816.618388:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204816.618390:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204816.618392:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204816.618394:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204816.618395:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204816.618397:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204816.618399:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618404:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.618406:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618410:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.618411:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204816.618413:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204816.618416:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204816.618418:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204816.618419:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300943920 00000100:00000001:3.0:1788204816.618421:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.618422:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300943920 00000100:00000001:3.0:1788204816.618424:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300947970 00000100:00000001:3.0:1788204816.618425:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204816.618427:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.618428:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204816.618430:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204816/real 1788204816] req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204816.618435:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204816.618436:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204816.618437:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204816.618438:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204816.618439:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204816.618440:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204816.618442:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204816.618449:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618456:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204816.618458:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618460:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618461:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204816.618465:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618471:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204816.618475:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618481:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204816.618484:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618485:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204816.618487:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618488:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204816.618491:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204816.618493:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204816.618497:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204816.618499:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204816.618503:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204816.618511:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205900800:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204816.618514:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618516:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.618518:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205900544/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618524:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204816.618525:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204816.618527:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204816.618528:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204816.618529:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204816.618530:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204816.618532:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204816.618533:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:3.0:1788204816.618535:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204816.618535:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204816.618536:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:3.0:1788204816.618538:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204816.618539:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618540:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.618542:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205900672/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618546:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204816.618546:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204816.618547:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204816.618549:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204816.618550:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204816.618550:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204816.618552:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204816.618554:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:3.0:1788204816.618556:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204816.618557:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204816.618558:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:3.0:1788204816.618561:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204816.618562:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618563:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204816.618566:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205900800/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204816.618572:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204816.618574:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204816.618576:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204816.618578:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204816.618579:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204816.618580:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204816.618583:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204816.618584:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:3.0:1788204816.618586:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204816.618587:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204816.618589:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:3.0:1788204816.618591:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204816.618592:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618594:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204816.618597:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204816.618599:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204816.618601:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204816.618603:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204816.808496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204816.808501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204816.808506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204816.808508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204816.808510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204816.808511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204816.808513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204816.808514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204816.808538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204816.808542:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204816.808545:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204817.128455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204817.128461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204817.128463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204817.128466:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204817.128469:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204817.128472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204817.128474:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204817.128475:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204817.128477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204817.576456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204817.576464:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204817.576466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204817.576472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204817.576475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204817.576479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204817.576481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204817.576484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204817.576486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204817.640481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204817.640481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204817.640489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204817.640490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204817.640492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204817.640492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204817.640499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204817.640500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204817.640504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204817.640505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204817.640508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204817.640509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204817.640511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204817.640515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204817.640518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204817.640518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204817.640522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204817.640525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204817.832538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204817.832543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204817.832546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204817.832547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204817.832549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204817.832550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204817.832552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204817.832552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204817.832566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204817.832569:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204817.832570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204818.152506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204818.152514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204818.152516:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204818.152521:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204818.152524:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204818.152528:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204818.152530:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204818.152533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204818.152536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204818.600470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204818.600478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204818.600480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204818.600485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204818.600488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204818.600492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204818.600495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204818.600497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204818.600500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204818.664518:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204818.664518:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204818.664529:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204818.664530:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204818.664533:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204818.664534:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204818.664543:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204818.664543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204818.664549:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204818.664550:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204818.664557:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204818.664557:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204818.664563:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204818.664568:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204818.664572:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204818.664573:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204818.664577:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204818.664582:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204818.856476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204818.856481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204818.856485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204818.856486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204818.856488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204818.856489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204818.856491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204818.856492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204818.856532:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204818.856539:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204818.856544:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204819.176519:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204819.176530:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204819.176532:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204819.176538:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204819.176541:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204819.176545:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204819.176548:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204819.176550:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204819.176553:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204819.624460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204819.624469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204819.624472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204819.624478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204819.624482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204819.624486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204819.624489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204819.624492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204819.624496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204819.688479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204819.688479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204819.688487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204819.688487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204819.688491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204819.688491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204819.688497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204819.688499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204819.688502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204819.688504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204819.688506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204819.688508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204819.688509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204819.688512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204819.688516:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204819.688516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204819.688520:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204819.688524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204819.880550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204819.880561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204819.880563:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204819.880566:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880569:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204819.880571:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880572:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880575:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880577:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204819.880578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204819.880581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204819.880582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204819.880596:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204819.880598:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204819.880600:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204820.200493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204820.200501:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204820.200503:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204820.200509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204820.200512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204820.200516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204820.200519:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204820.200521:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204820.200523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204820.648458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204820.648468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204820.648470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204820.648477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204820.648482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204820.648487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204820.648489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204820.648492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204820.648495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204820.712495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204820.712495:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204820.712507:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204820.712507:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204820.712511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204820.712511:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204820.712520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204820.712520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204820.712527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204820.712527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204820.712534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204820.712534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204820.712540:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204820.712544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204820.712547:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204820.712548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204820.712552:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204820.712556:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204820.904488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204820.904497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204820.904505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204820.904506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204820.904509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204820.904510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204820.904513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204820.904514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204820.904552:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204820.904557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204820.904561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204821.224468:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300947970, at 4300948480 00000100:00000040:1.0:1788204821.224468:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300947970, at 4300948480 00000100:00000001:2.0:1788204821.224468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204821.224477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204821.224483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204821.224488:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204821.224489:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000010:1.0:1788204821.224491:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000001:2.0:1788204821.224492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204821.224498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204821.224502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204821.224505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204821.224509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204821.672521:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204821.672528:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204821.672530:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204821.672535:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204821.672538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204821.672541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204821.672544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204821.672546:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204821.672548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204821.736495:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204821.736497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204821.736498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204821.736504:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204821.736505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204821.736508:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204821.736509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.736509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736515:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204821.736517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204821.736517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204821.736521:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204821.736523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204821.736524:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204821.736529:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.736529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204821.736530:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204821.736533:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204821.736536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204821.736538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.736540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204821.736542:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6276 00000100:00000001:3.0:1788204821.736544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.736549:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204821.736557:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204821.736565:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204821.736569:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204821.736572:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736575:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736577:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736578:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204821.736582:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204821.736583:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204821.736586:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204821.736589:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204821.736590:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204821.736592:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204821.736594:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736595:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736597:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736598:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204821.736600:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204821.736602:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736604:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204821.736606:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204821.736620:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204821.736622:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736624:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204821.736627:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204821.736637:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204821.736640:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204821.736643:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204821.736644:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736647:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204821.736649:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204821.736653:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204821.736663:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204821.736669:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204821.736679:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204821.736687:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204821.736691:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204821.736695:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204821.736711:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204821.736713:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204821.736716:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204821.736717:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204821.736718:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204821.736720:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204821.736721:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6276 00000400:00000200:0.0:1788204821.736723:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204821.736726:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204821.736727:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204821.736730:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736732:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736733:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736735:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204821.736736:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204821.736737:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204821.736738:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204821.736738:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1788204821.736741:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204821.736743:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204821.736744:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204821.736747:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204821.736751:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204821.736754:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736755:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736756:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736758:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204821.736760:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204821.736763:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736765:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204821.736765:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204821.736769:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204821.736774:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204821.736779:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736781:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204821.736781:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204821.736785:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204821.736788:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204821.736791:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1788204821.736793:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.736796:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204821.736797:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.736801:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204821.736804:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00100000:3.0:1788204821.736809:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205900928:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788204821.736810:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204821.736815:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204821.736818:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204821.736819:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736823:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204821.736824:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204821.736825:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204821.736829:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204821.736833:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.736833:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204821.736840:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204821.736845:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204821.736849:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 00000100:00100000:0.0:1788204821.736851:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204821.736855:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.736865:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204821.736866:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204821.736869:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204821.736873:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000040:0.0:1788204821.736876:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204821.736885:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:0.0:1788204821.736887:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204821.736891:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204821.736895:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204821.736896:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204821.736898:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204821.736902:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788204821.736902:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205900928, portal 25 00000100:00080000:0.0:1788204821.736906:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6276 00000100:00000001:3.0:1788204821.736908:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204821.736912:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000400:00000200:0.0:1788204821.736916:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204821.736923:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:3.0:1788204821.736923:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204821.736927:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204821.736936:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204821.736940:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204821.736943:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000040:0.0:1788204821.736948:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204821.736949:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205900928, offset 0 00000100:00000001:0.0:1788204821.736953:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736957:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736961:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204821.736965:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788204821.736965:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788204821.736966:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204821.736970:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204821.736972:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204821.736975:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204821.736978:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204821.736980:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204821.736985:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.736988:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.736993:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204821.736993:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204821.736997:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204821.737001:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204821.737003:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204821.737005:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204821.737006:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204821.737009:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204821.737011:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204821.737012:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:0.0:1788204821.737013:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.737016:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204821.737018:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204821.737019:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000008:00000001:0.0:1788204821.737023:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737026:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737027:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204821.737032:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000200:3.0:1788204821.737036:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58894:58894:256:4294967295] 192.168.201.151@tcp LPNI seq info [58874:58874:8:4294967295] 00000020:00000040:0.0:1788204821.737037:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204821.737041:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204821.737043:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.737045:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204821.737046:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204821.737048:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737049:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:0.0:1788204821.737053:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204821.737058:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204821.737065:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204821.737070:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204821.737084:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:3.0:1788204821.737084:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737088:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204821.737093:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:0.0:1788204821.737095:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788204821.737107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204821.737108:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204821.737112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00080000:0.0:1788204821.737113:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204821.737120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204821.737121:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204821.737126:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788204821.737127:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00080000:0.0:1788204821.737130:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788204821.737134:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737136:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:3.0:1788204821.737139:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00080000:0.0:1788204821.737140:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6276 00000400:00000200:0.0:1788204821.737146:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204821.737149:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204821.737151:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204821.737152:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1788204821.737154:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737157:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.737161:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737164:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204821.737165:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1788204821.737167:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204821.737169:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204821.737172:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204821.737174:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204821.737175:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204821.737176:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:3.0:1788204821.737177:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000100:00000040:0.0:1788204821.737180:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204821.737182:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:0.0:1788204821.737183:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204821.737187:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737189:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204821.737190:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204821.737191:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1788204821.737193:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204821.737196:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204821.737197:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204821.737198:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204821.737201:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204821.737205:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.737209:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204821.737212:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204821.737214:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737215:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204821.737216:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204821.737220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788204821.737222:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204821.737226:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204821.737231:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204821.737232:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204821.737233:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.737237:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204821.737237:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1788204821.737240:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204821.737243:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204821.737246:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:3.0:1788204821.737247:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204821.737251:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737253:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204821.737258:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300953089 00000100:00100000:0.0:1788204821.737263:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788204821.737265:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f48a0. 00000100:00000001:0.0:1788204821.737274:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204821.737275:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204821.737277:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204821.737284:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737285:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300953089 00000100:00000010:3.0:1788204821.737292:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4a20. 00000100:00000001:3.0:1788204821.737298:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204821.737299:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737304:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788204821.737308:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6286) 00000100:00000001:3.0:1788204821.737309:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737314:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204821.737320:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204821/real 1788204821] req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204821.737329:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204821.737330:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204821.737332:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204821.737334:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204821.737336:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204821.737338:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204821.737340:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204821.737346:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737350:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204821.737351:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737353:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737354:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204821.737356:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737361:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204821.737382:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737387:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204821.737389:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737390:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737391:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737393:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204821.737394:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204821.737396:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204821.737398:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204821.737400:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204821.737403:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204821.737409:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205900928:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204821.737412:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737413:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.737415:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205900928/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737418:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204821.737420:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204821.737422:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204821.737425:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204821.737426:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204821.737427:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204821.737429:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204821.737431:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788204821.737434:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204821.737435:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204821.737437:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204821.737439:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204821.737440:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737441:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737448:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204821.737449:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737451:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204821.737452:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204821.737453:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204821.737455:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737459:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204821.737461:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737462:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204821.737463:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204821.737465:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204821.737467:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901056:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204821.737469:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204821.737471:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204821.737472:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737473:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204821.737475:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204821.737476:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737479:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204821.737480:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204821.737483:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204821.737485:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901056, portal 10 00000100:00000001:3.0:1788204821.737486:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204821.737487:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204821.737492:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737496:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204821.737498:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204821.737499:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204821.737500:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205901056, offset 0 00000400:00000200:3.0:1788204821.737502:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204821.737505:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.737508:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204821.737511:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204821.737513:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204821.737516:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204821.737518:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204821.737521:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58895:58895:256:4294967295] 192.168.201.151@tcp LPNI seq info [58875:58875:8:4294967295] 00000400:00000200:3.0:1788204821.737525:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.737531:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204821.737536:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.737539:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204821.737543:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204821.737546:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204821.737550:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204821.737572:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204821.737576:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.737578:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204821.737581:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204821.737584:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204821.737588:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204821.737590:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204821.737592:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204821.737596:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737604:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.737619:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737625:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737627:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204821.737631:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204821.737635:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204821.737638:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204821.737641:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300949147 00000100:00000001:3.0:1788204821.737644:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.737646:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300949147 00000100:00000001:3.0:1788204821.737649:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.737651:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204821.737653:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737655:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204821.737659:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204821/real 1788204821] req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204821.737674:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204821.737675:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204821.737677:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204821.737679:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204821.737681:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204821.737683:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204821.737686:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204821.737694:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737701:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204821.737702:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737704:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737705:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204821.737707:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737712:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204821.737715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737721:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204821.737724:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737725:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737726:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737728:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204821.737730:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204821.737733:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204821.737736:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204821.737738:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204821.737743:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204821.737751:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901056:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204821.737755:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204821.737758:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737763:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204821.737764:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737766:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204821.737768:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204821.737769:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204821.737771:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901184:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204821.737773:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204821.737774:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204821.737783:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737785:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204821.737787:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204821.737791:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737793:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204821.737795:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204821.737798:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204821.737800:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901184, portal 4 00000100:00000001:3.0:1788204821.737801:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204821.737802:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204821.737807:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737811:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204821.737812:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204821.737814:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204821.737815:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205901184, offset 0 00000400:00000200:3.0:1788204821.737818:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204821.737820:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.737823:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204821.737825:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204821.737827:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204821.737828:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204821.737830:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204821.737832:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58896:58896:256:4294967295] 192.168.201.151@tcp LPNI seq info [58876:58876:8:4294967295] 00000400:00000200:3.0:1788204821.737835:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.737839:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204821.737842:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.737843:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204821.737846:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204821.737848:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204821.737850:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204821.737852:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204821.737855:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.737856:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204821.737858:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204821.737860:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204821.737862:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204821.737863:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204821.737864:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204821.737866:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737872:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.737873:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737877:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737878:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204821.737880:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204821.737882:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204821.737884:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204821.737886:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300949033 00000100:00000001:3.0:1788204821.737887:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.737889:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300949033 00000100:00000001:3.0:1788204821.737890:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.737891:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204821.737892:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.737893:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204821.737895:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204821/real 1788204821] req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204821.737901:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204821.737902:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204821.737904:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204821.737906:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204821.737908:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204821.737909:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204821.737912:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204821.737920:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737926:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204821.737928:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737930:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737931:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204821.737934:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737941:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204821.737945:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737950:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204821.737952:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737953:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.737954:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.737955:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204821.737957:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204821.737959:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204821.737961:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204821.737963:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204821.737965:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204821.737969:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901184:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204821.737971:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204821.737973:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.737976:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204821.737977:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737978:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204821.737979:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204821.737980:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204821.737982:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901312:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204821.737984:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204821.737985:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204821.737986:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204821.737987:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204821.737990:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204821.737991:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204821.737993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204821.737994:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204821.737996:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204821.737998:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901312, portal 4 00000100:00000001:3.0:1788204821.738000:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204821.738001:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204821.738005:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738009:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204821.738010:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204821.738011:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204821.738021:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205901312, offset 0 00000400:00000200:3.0:1788204821.738024:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204821.738030:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.738034:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204821.738036:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204821.738039:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204821.738041:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204821.738043:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204821.738046:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58897:58897:256:4294967295] 192.168.201.151@tcp LPNI seq info [58877:58877:8:4294967295] 00000400:00000200:3.0:1788204821.738050:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204821.738056:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204821.738060:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.738063:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204821.738067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204821.738070:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204821.738073:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204821.738076:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204821.738080:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204821.738081:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204821.738084:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204821.738087:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204821.738090:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204821.738092:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204821.738094:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204821.738097:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738107:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.738109:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738113:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.738114:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204821.738117:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204821.738119:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204821.738121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204821.738122:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300949039 00000100:00000001:3.0:1788204821.738124:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.738125:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300949039 00000100:00000001:3.0:1788204821.738127:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300953089 00000100:00000001:3.0:1788204821.738128:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204821.738130:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.738131:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204821.738134:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204821/real 1788204821] req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204821.738142:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204821.738143:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204821.738145:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204821.738147:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204821.738148:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204821.738151:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204821.738154:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204821.738162:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738169:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204821.738171:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738173:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.738174:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204821.738177:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738182:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204821.738184:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738189:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204821.738190:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738191:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204821.738192:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738193:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204821.738194:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204821.738196:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204821.738199:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204821.738201:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204821.738204:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204821.738211:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901312:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204821.738215:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738217:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.738220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205901056/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738227:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204821.738229:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204821.738231:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204821.738234:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204821.738235:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204821.738236:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204821.738238:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204821.738240:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:3.0:1788204821.738241:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204821.738243:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204821.738244:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204821.738245:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204821.738246:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738248:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.738250:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205901184/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738254:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204821.738255:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204821.738256:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204821.738257:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204821.738258:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204821.738258:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204821.738259:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204821.738260:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788204821.738261:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204821.738262:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204821.738263:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204821.738264:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204821.738265:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738266:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204821.738267:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205901312/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204821.738270:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204821.738271:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204821.738272:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204821.738273:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204821.738274:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204821.738275:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204821.738276:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204821.738277:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788204821.738280:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204821.738282:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204821.738283:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204821.738286:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204821.738287:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738289:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204821.738293:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204821.738294:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204821.738297:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204821.738298:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204821.928538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204821.928545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204821.928550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204821.928551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204821.928553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204821.928554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204821.928556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204821.928557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204821.928572:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204821.928573:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204821.928575:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204822.248491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204822.248498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204822.248500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204822.248505:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204822.248509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204822.248512:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204822.248515:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204822.248517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204822.248519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204822.696485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204822.696493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204822.696495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204822.696500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204822.696503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204822.696507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204822.696510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204822.696512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204822.696514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204822.760483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204822.760483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204822.760494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204822.760494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204822.760497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204822.760498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204822.760503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204822.760504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204822.760508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204822.760509:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204822.760513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204822.760514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204822.760516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204822.760519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204822.760522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204822.760525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204822.760528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204822.760532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204822.952519:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204822.952523:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204822.952527:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952533:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204822.952536:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952539:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952542:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204822.952549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204822.952555:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204822.952556:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204822.952558:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952560:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204822.952561:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952563:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952565:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204822.952570:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204822.952571:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204822.952573:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952575:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204822.952576:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952578:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204822.952580:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204822.952647:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204822.952653:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204822.952657:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204823.272511:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204823.272523:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204823.272525:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.272531:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204823.272536:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204823.272541:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204823.272544:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204823.272548:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204823.272552:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204823.720496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204823.720507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204823.720509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204823.720516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204823.720519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204823.720523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204823.720525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204823.720527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204823.720530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204823.784485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204823.784485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204823.784496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204823.784496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204823.784501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204823.784501:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204823.784509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204823.784509:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204823.784516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204823.784516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204823.784521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204823.784522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204823.784527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204823.784530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204823.784532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204823.784535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204823.784538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204823.784541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204823.976499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204823.976508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204823.976515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204823.976516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204823.976518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204823.976519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204823.976522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204823.976523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204823.976539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204823.976541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204823.976550:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204824.296478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204824.296487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204824.296490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204824.296496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204824.296499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204824.296503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204824.296505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204824.296508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204824.296511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204824.744474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204824.744482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204824.744484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204824.744489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204824.744492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204824.744496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204824.744498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204824.744500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204824.744502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204824.808441:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204824.808451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204824.808453:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204824.808460:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204824.808464:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204824.808468:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204824.808471:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204824.808474:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204824.808478:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204824.808499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204824.808506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204824.808508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204824.808513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204824.808516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204824.808521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204824.808523:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204824.808526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204824.808529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204825.000489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204825.000498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204825.000505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204825.000506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204825.000515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204825.000518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204825.000522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204825.000523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204825.000604:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204825.000609:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204825.000613:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204825.320449:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204825.320457:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204825.320458:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204825.320464:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204825.320467:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204825.320470:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204825.320473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204825.320476:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204825.320478:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204825.768474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204825.768481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204825.768483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204825.768488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204825.768491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204825.768495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204825.768497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204825.768499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204825.768502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204825.832470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204825.832470:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204825.832478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204825.832479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204825.832482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204825.832482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204825.832489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204825.832489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204825.832495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204825.832495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204825.832500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204825.832500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204825.832505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204825.832508:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204825.832510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204825.832510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204825.832513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204825.832515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204826.024488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204826.024494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204826.024498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204826.024499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204826.024501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204826.024502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204826.024504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204826.024505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204826.024518:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204826.024520:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204826.024523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204826.344493:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300953089, at 4300953600 00000100:00000040:1.0:1788204826.344493:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300953089, at 4300953600 00000100:00000001:2.0:1788204826.344496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204826.344503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204826.344507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204826.344509:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000010:0.0:1788204826.344510:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00100000:2.0:1788204826.344512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204826.344518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204826.344522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204826.344525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204826.344553:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204826.344558:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204826.792469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204826.792476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204826.792478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204826.792483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204826.792486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204826.792490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204826.792492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204826.792494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204826.792496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204826.856483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204826.856483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204826.856492:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204826.856493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204826.856497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.856498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204826.856499:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204826.856501:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856504:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788204826.856506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204826.856506:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204826.856508:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204826.856512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204826.856513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204826.856515:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204826.856517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204826.856532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.856533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204826.856536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204826.856538:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6281 00000100:00000001:1.0:1788204826.856541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204826.856541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204826.856545:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204826.856550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204826.856552:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204826.856562:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204826.856566:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204826.856575:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856578:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.856580:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856582:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204826.856585:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204826.856587:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204826.856589:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204826.856592:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204826.856594:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204826.856595:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204826.856597:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856600:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.856601:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856604:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204826.856607:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204826.856610:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856613:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204826.856619:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204826.856623:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204826.856626:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856629:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204826.856632:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204826.856644:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204826.856648:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204826.856652:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204826.856653:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856655:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204826.856658:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204826.856662:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204826.856670:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204826.856676:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204826.856684:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204826.856692:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204826.856695:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204826.856699:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204826.856725:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204826.856728:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204826.856731:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204826.856733:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204826.856735:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204826.856738:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204826.856743:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6281 00000400:00000200:0.0:1788204826.856748:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204826.856749:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1788204826.856753:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204826.856755:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204826.856757:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204826.856759:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204826.856763:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856768:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204826.856769:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204826.856772:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856777:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204826.856782:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204826.856785:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788204826.856786:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204826.856789:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204826.856790:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204826.856793:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:3.0:1788204826.856796:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788204826.856797:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204826.856799:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788204826.856802:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204826.856803:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204826.856808:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205901440:192.168.201.151@tcp:250:kworker.0 00000100:00000040:0.0:1788204826.856809:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856814:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.856816:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204826.856816:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204826.856819:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204826.856821:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204826.856821:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788204826.856825:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204826.856825:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856830:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204826.856831:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204826.856831:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204826.856838:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204826.856840:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856841:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:3.0:1788204826.856841:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204826.856845:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204826.856846:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204826.856849:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856851:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.856855:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:0.0:1788204826.856856:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204826.856859:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204826.856861:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788204826.856862:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 02000000:00000001:0.0:1788204826.856863:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.856867:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204826.856868:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204826.856870:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204826.856873:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 02000000:00000010:0.0:1788204826.856874:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788204826.856880:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204826.856884:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204826.856884:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901440, portal 25 00000100:00000001:3.0:1788204826.856890:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204826.856892:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204826.856894:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204826.856897:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204826.856900:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204826.856901:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.856924:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204826.856925:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204826.856929:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204826.856930:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204826.856933:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00080000:0.0:1788204826.856934:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204826.856936:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204826.856938:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788204826.856938:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205901440, offset 0 00000100:00000001:0.0:1788204826.856943:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204826.856946:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6281 00000400:00000200:3.0:1788204826.856949:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204826.856953:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204826.856956:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204826.856958:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204826.856962:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204826.856962:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204826.856966:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.856968:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856970:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204826.856972:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:0.0:1788204826.856973:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204826.856975:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204826.856977:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204826.856979:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204826.856980:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1788204826.856981:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204826.856984:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204826.856986:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:0.0:1788204826.856987:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.856991:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204826.856992:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204826.856997:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204826.856997:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.857002:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204826.857005:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58898:58898:256:4294967295] 192.168.201.151@tcp LPNI seq info [58878:58878:8:4294967295] 00000020:00000040:0.0:1788204826.857007:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204826.857014:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857015:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204826.857027:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.857031:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 02000000:00000001:0.0:1788204826.857033:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204826.857038:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204826.857043:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204826.857044:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204826.857046:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204826.857050:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000010:3.0:1788204826.857051:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000008:00000001:0.0:1788204826.857056:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.857059:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204826.857063:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000800:00000200:3.0:1788204826.857064:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204826.857069:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.0:1788204826.857071:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 02000000:00000001:0.0:1788204826.857073:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204826.857077:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.857078:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204826.857081:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788204826.857082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204826.857083:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204826.857088:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000800:00000200:3.1:1788204826.857089:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1788204826.857092:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204826.857099:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204826.857100:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.857106:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000001:0.0:1788204826.857108:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204826.857110:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204826.857113:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204826.857116:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204826.857123:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.857127:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1788204826.857128:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204826.857134:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204826.857134:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204826.857135:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204826.857137:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204826.857138:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000100:00080000:0.0:1788204826.857139:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6281 00000400:00000200:0.0:1788204826.857144:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204826.857144:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000400:00000200:0.0:1788204826.857149:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204826.857151:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204826.857152:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1788204826.857153:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.857157:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.857160:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204826.857161:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204826.857162:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204826.857164:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204826.857166:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204826.857168:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204826.857171:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204826.857174:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204826.857178:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204826.857185:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204826.857189:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204826.857192:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204826.857193:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204826.857194:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204826.857198:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000040:3.0:1788204826.857198:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204826.857201:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204826.857205:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204826.857208:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204826.857211:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204826.857212:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204826.857215:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857216:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204826.857219:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204826.857221:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204826.857223:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.857224:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857225:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204826.857227:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204826.857230:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:3.0:1788204826.857230:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204826.857235:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204826.857236:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300958209 02000000:00000001:0.0:1788204826.857237:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204826.857240:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204826.857241:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204826.857242:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204826.857243:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f41e0. 02000000:00000010:0.0:1788204826.857245:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204826.857248:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857251:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1788204826.857252:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204826.857259:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204826.857259:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300958209 00000100:00000001:0.0:1788204826.857262:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204826.857264:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204826.857268:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204826.857273:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6291) 00000100:00000010:3.0:1788204826.857275:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f44e0. 00000100:00000001:3.0:1788204826.857283:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204826.857286:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204826.857288:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857290:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204826.857295:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204826/real 1788204826] req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204826.857302:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204826.857303:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204826.857305:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204826.857307:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204826.857309:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204826.857311:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204826.857314:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204826.857319:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857323:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204826.857325:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857326:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857328:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204826.857331:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857335:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204826.857338:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857342:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204826.857344:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857345:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857346:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857347:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204826.857350:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204826.857352:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204826.857354:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204826.857355:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204826.857358:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204826.857379:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205901440:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204826.857383:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857384:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.857386:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205901440/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857391:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204826.857393:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204826.857395:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204826.857397:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204826.857398:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204826.857400:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204826.857402:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204826.857404:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788204826.857407:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204826.857409:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204826.857410:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204826.857412:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204826.857413:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857415:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857421:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204826.857422:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857424:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204826.857425:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204826.857426:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204826.857428:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857433:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204826.857434:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857435:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204826.857436:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204826.857438:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204826.857439:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901568:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204826.857441:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204826.857442:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204826.857443:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857444:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204826.857446:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204826.857448:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.857452:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204826.857454:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204826.857457:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204826.857460:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901568, portal 10 00000100:00000001:3.0:1788204826.857462:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204826.857463:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204826.857470:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857477:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204826.857479:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204826.857481:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204826.857483:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205901568, offset 0 00000400:00000200:3.0:1788204826.857487:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857491:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.857494:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204826.857497:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204826.857499:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204826.857502:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204826.857504:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857507:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58899:58899:256:4294967295] 192.168.201.151@tcp LPNI seq info [58879:58879:8:4294967295] 00000400:00000200:3.0:1788204826.857511:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.857517:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204826.857537:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.857541:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204826.857545:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204826.857548:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204826.857551:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204826.857552:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204826.857555:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.857556:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204826.857558:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204826.857561:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204826.857563:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204826.857564:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204826.857566:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204826.857568:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857573:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.857575:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857583:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857584:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204826.857586:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204826.857590:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204826.857592:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204826.857594:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300954267 00000100:00000001:3.0:1788204826.857596:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.857597:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300954267 00000100:00000001:3.0:1788204826.857599:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.857600:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204826.857601:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857602:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204826.857604:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204826/real 1788204826] req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204826.857609:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204826.857610:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204826.857611:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204826.857613:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204826.857614:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204826.857615:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204826.857617:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204826.857621:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857625:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204826.857625:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857626:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857627:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204826.857629:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857634:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204826.857636:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857640:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204826.857641:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857642:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857643:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857644:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204826.857645:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204826.857646:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204826.857648:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204826.857650:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204826.857653:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204826.857660:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901568:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204826.857664:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204826.857667:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857674:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204826.857675:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857677:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204826.857679:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204826.857681:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204826.857684:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901696:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204826.857687:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204826.857689:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204826.857691:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857693:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204826.857697:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204826.857699:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.857702:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204826.857705:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204826.857708:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204826.857711:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901696, portal 4 00000100:00000001:3.0:1788204826.857713:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204826.857715:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204826.857720:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857727:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204826.857729:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204826.857731:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204826.857733:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205901696, offset 0 00000400:00000200:3.0:1788204826.857737:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857740:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.857743:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204826.857746:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204826.857749:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204826.857751:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204826.857753:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857756:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58900:58900:256:4294967295] 192.168.201.151@tcp LPNI seq info [58880:58880:8:4294967295] 00000400:00000200:3.0:1788204826.857760:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.857766:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204826.857770:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.857774:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204826.857777:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204826.857780:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204826.857784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204826.857786:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204826.857790:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.857793:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204826.857796:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204826.857799:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204826.857802:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204826.857804:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204826.857806:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204826.857809:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857817:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.857821:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857827:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857829:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204826.857832:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204826.857835:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204826.857838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204826.857840:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300954153 00000100:00000001:3.0:1788204826.857842:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.857844:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300954153 00000100:00000001:3.0:1788204826.857846:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.857848:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204826.857850:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.857852:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204826.857856:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204826/real 1788204826] req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204826.857863:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204826.857864:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204826.857866:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204826.857868:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204826.857869:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204826.857872:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204826.857875:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204826.857883:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857887:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204826.857888:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857890:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857891:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204826.857893:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857897:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204826.857899:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857903:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204826.857904:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857905:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.857906:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.857907:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204826.857908:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204826.857910:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204826.857911:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204826.857912:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204826.857914:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204826.857918:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901696:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204826.857921:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204826.857923:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857927:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204826.857928:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857930:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204826.857931:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204826.857932:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204826.857934:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901824:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204826.857935:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204826.857936:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204826.857937:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204826.857938:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204826.857941:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204826.857942:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204826.857944:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204826.857946:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204826.857948:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204826.857949:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901824, portal 4 00000100:00000001:3.0:1788204826.857951:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204826.857952:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204826.857956:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.857963:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204826.857965:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204826.857967:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204826.857970:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205901824, offset 0 00000400:00000200:3.0:1788204826.857974:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857978:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.857981:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204826.857984:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204826.857986:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204826.857988:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204826.857991:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204826.857994:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58901:58901:256:4294967295] 192.168.201.151@tcp LPNI seq info [58881:58881:8:4294967295] 00000400:00000200:3.0:1788204826.857998:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204826.858004:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204826.858009:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.858012:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204826.858015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204826.858018:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204826.858033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204826.858036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204826.858039:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204826.858041:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204826.858044:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204826.858048:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204826.858050:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204826.858051:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204826.858053:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204826.858055:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858061:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.858062:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858066:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.858067:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204826.858069:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204826.858071:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204826.858073:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204826.858075:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300954159 00000100:00000001:3.0:1788204826.858076:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.858078:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300954159 00000100:00000001:3.0:1788204826.858079:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300958209 00000100:00000001:3.0:1788204826.858080:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204826.858082:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.858083:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204826.858085:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204826/real 1788204826] req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204826.858090:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204826.858091:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204826.858092:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204826.858093:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204826.858094:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204826.858095:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204826.858096:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204826.858101:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858105:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204826.858105:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858106:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.858107:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204826.858109:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858112:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204826.858114:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858118:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204826.858119:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858120:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204826.858121:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858122:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204826.858124:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204826.858125:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204826.858128:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204826.858129:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204826.858131:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204826.858136:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205901824:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204826.858137:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858138:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.858140:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205901568/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858143:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204826.858144:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204826.858145:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204826.858147:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204826.858148:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204826.858148:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204826.858150:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204826.858151:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788204826.858153:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204826.858154:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204826.858155:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204826.858157:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204826.858157:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858158:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.858160:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205901696/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858163:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204826.858164:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204826.858165:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204826.858166:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204826.858166:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204826.858167:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204826.858168:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204826.858169:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788204826.858171:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204826.858171:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204826.858172:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204826.858174:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204826.858174:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858175:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204826.858177:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205901824/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204826.858180:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204826.858180:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204826.858181:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204826.858182:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204826.858183:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204826.858183:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204826.858184:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204826.858185:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:3.0:1788204826.858186:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204826.858187:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204826.858187:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204826.858188:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204826.858189:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858190:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204826.858192:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204826.858193:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204826.858194:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204826.858195:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204827.048492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204827.048498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204827.048504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204827.048506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204827.048508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204827.048509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204827.048513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204827.048539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204827.048601:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204827.048605:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204827.048610:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204827.368527:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204827.368538:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204827.368540:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204827.368546:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204827.368550:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204827.368553:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204827.368556:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204827.368558:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204827.368561:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204827.816469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204827.816474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204827.816476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204827.816480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204827.816482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204827.816485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204827.816487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204827.816489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204827.816491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204827.880455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204827.880455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204827.880461:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204827.880462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204827.880464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204827.880465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204827.880470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204827.880471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204827.880474:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204827.880477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204827.880478:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204827.880481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204827.880482:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204827.880484:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204827.880487:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204827.880488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204827.880491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204827.880494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204828.072554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204828.072561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204828.072566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204828.072567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204828.072571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204828.072572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204828.072575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204828.072576:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204828.072596:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204828.072599:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204828.072602:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204828.392463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204828.392469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204828.392470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.392473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204828.392476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204828.392478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204828.392480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204828.392482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204828.392484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204828.840482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204828.840499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204828.840501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204828.840506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204828.840510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204828.840513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204828.840516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204828.840518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204828.840521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204828.904481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204828.904481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204828.904501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204828.904504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204828.904507:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204828.904511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204828.904513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204828.904516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204828.904519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204828.904524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204828.904525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204828.904526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204828.904529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204828.904534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204828.904538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204828.904538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204828.904541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204828.904546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204829.096500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204829.096509:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204829.096512:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204829.096516:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096519:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204829.096521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096523:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096525:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204829.096530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204829.096533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204829.096535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204829.096582:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204829.096585:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204829.096588:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204829.416461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204829.416467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204829.416468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204829.416473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204829.416496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204829.416499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204829.416500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204829.416502:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204829.416504:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204829.864433:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204829.864441:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204829.864442:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204829.864447:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204829.864449:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204829.864452:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204829.864454:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204829.864457:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204829.864460:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204829.928448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204829.928448:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204829.928456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204829.928457:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204829.928461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204829.928461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204829.928476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204829.928477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204829.928481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204829.928482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204829.928487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204829.928488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204829.928493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204829.928496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204829.928498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204829.928500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204829.928503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204829.928506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204830.120504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204830.120509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204830.120512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204830.120513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204830.120515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204830.120516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204830.120518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204830.120519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204830.120532:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204830.120533:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204830.120535:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204830.440519:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204830.440526:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204830.440528:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204830.440534:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204830.440537:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204830.440541:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204830.440544:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204830.440547:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204830.440549:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204830.888489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204830.888496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204830.888497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204830.888501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204830.888509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204830.888514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204830.888516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204830.888517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204830.888519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204830.952475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204830.952475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204830.952482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204830.952482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204830.952486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204830.952486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204830.952491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204830.952492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204830.952495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204830.952496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204830.952500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204830.952500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204830.952505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204830.952508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204830.952510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204830.952510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204830.952513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204830.952515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204831.144489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204831.144496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204831.144502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204831.144504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204831.144507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204831.144509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204831.144513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204831.144514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204831.144578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204831.144581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204831.144584:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204831.464438:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300958209, at 4300958720 00000100:00000040:1.0:1788204831.464454:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300958209, at 4300958720 00000100:00000001:2.0:1788204831.464455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204831.464461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788204831.464463:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000010:1.0:1788204831.464464:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000001:2.0:1788204831.464464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204831.464470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204831.464474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204831.464477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.464480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204831.464483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.464486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204831.912470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204831.912476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204831.912478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204831.912484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204831.912487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204831.912491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204831.912493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204831.912496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204831.912499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204831.976455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204831.976463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204831.976465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204831.976467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204831.976470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204831.976472:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788204831.976473:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204831.976476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204831.976477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788204831.976482:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1788204831.976483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204831.976483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204831.976489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204831.976489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204831.976490:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204831.976493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204831.976494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788204831.976495:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204831.976501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204831.976502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976502:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204831.976506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204831.976511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204831.976512:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6286 00000400:00000200:3.0:1788204831.976522:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204831.976528:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204831.976531:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204831.976535:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976538:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.976540:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976542:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204831.976546:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204831.976547:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204831.976550:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204831.976553:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204831.976555:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204831.976558:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204831.976560:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976562:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.976564:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976569:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204831.976572:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204831.976575:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976577:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204831.976579:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204831.976583:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204831.976585:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976587:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204831.976590:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204831.976597:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1c00. 00000020:00000040:3.0:1788204831.976601:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204831.976605:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204831.976606:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976610:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204831.976612:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204831.976615:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204831.976623:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23000. 00000100:00000001:3.0:1788204831.976629:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204831.976638:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204831.976649:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204831.976652:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204831.976655:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004207d831] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204831.976674:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204831.976676:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204831.976678:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204831.976679:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204831.976681:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204831.976683:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204831.976684:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6286 00000400:00000200:3.0:1788204831.976687:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204831.976689:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204831.976691:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204831.976696:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976697:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.976698:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976700:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204831.976701:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204831.976702:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204831.976703:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204831.976705:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788204831.976706:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788204831.976708:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204831.976710:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976712:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.976714:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976715:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204831.976717:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788204831.976719:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204831.976721:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204831.976721:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204831.976725:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204831.976726:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204831.976728:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788204831.976729:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204831.976732:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976734:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:2.0:1788204831.976735:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788204831.976738:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204831.976743:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976744:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204831.976749:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3800. 00000100:00000001:2.0:1788204831.976750:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1788204831.976753:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:2.0:1788204831.976754:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204831.976757:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204831.976759:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204831.976759:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.976762:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204831.976762:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204831.976765:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204831.976766:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204831.976768:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204831.976771:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205901952:192.168.201.151@tcp:250:kworker.0 02000000:00000010:3.0:1788204831.976773:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21400. 00000100:00000001:2.0:1788204831.976775:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788204831.976778:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.976780:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204831.976783:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204831.976785:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204831.976786:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788204831.976792:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204831.976793:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204831.976794:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 00000100:00000040:3.0:1788204831.976796:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3172b36] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204831.976797:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.976806:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204831.976806:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204831.976809:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1788204831.976811:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000100:00080000:3.0:1788204831.976813:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204831.976815:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204831.976816:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1788204831.976817:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204831.976818:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204831.976822:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6286 00000100:00000200:2.0:1788204831.976825:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205901952, portal 25 00000400:00000200:3.0:1788204831.976825:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204831.976830:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1788204831.976830:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204831.976832:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300828672 : -113511408722944 : ffff98c310ce1c00) 00000100:00080000:3.0:1788204831.976833:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204831.976838:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976841:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204831.976842:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204831.976845:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976850:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204831.976854:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204831.976856:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000001:2.0:1788204831.976857:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788204831.976858:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204831.976860:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000040:2.0:1788204831.976862:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000001:3.0:1788204831.976863:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000010:2.0:1788204831.976865:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000020:00000001:3.0:1788204831.976866:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:2.0:1788204831.976869:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205901952, offset 0 00000100:00000040:3.0:1788204831.976871:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976874:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.976877:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204831.976879:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976879:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204831.976883:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788204831.976885:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976887:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.976890:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788204831.976890:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204831.976894:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204831.976896:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976898:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204831.976899:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00100000:3.0:1788204831.976900:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788204831.976904:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.976906:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204831.976906:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.976910:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000010:3.0:1788204831.976911:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2680. 00000400:00000200:2.0:1788204831.976914:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1788204831.976914:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788204831.976917:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1788204831.976919:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:3.0:1788204831.976920:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.976923:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1788204831.976925:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58902:58902:256:4294967295] 192.168.201.151@tcp LPNI seq info [58882:58882:8:4294967295] 02000000:00000001:3.0:1788204831.976925:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204831.976928:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.976932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:3.0:1788204831.976932:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22800. 00000100:00000001:3.0:1788204831.976936:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204831.976942:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204831.976945:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:3.0:1788204831.976955:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.0:1788204831.976958:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204831.976958:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204831.976964:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bb9116c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1788204831.976966:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000100:00000001:3.0:1788204831.976969:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204831.976972:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204831.976974:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204831.976976:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.0:1788204831.976978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:3.0:1788204831.976978:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204831.976982:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204831.976986:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6286 00000800:00000200:2.0:1788204831.976987:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000400:00000200:3.0:1788204831.976989:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204831.976994:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204831.976996:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1788204831.977000:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:3.0:1788204831.977001:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204831.977004:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.977006:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204831.977007:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1788204831.977008:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204831.977012:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204831.977014:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788204831.977016:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204831.977018:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000100:2.0:1788204831.977019:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:3.0:1788204831.977020:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000010:2.0:1788204831.977023:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000020:00000001:3.0:1788204831.977024:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204831.977030:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204831.977035:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1788204831.977039:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977040:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204831.977043:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:3.0:1788204831.977043:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204831.977048:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204831.977049:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788204831.977053:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:2.0:1788204831.977055:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204831.977058:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788204831.977066:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204831.977067:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204831.977070:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204831.977073:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1788204831.977077:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204831.977077:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204831.977083:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.977087:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204831.977090:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788204831.977090:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:2.0:1788204831.977095:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1788204831.977095:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204831.977099:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977103:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204831.977103:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0e00. 00000020:00000040:3.0:1788204831.977105:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:2.0:1788204831.977109:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788204831.977110:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204831.977112:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977113:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300963329 00000100:00000001:3.0:1788204831.977114:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204831.977116:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204831.977119:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204831.977120:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c10c0. 02000000:00000010:3.0:1788204831.977123:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20400. 00000100:00000001:2.0:1788204831.977127:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204831.977128:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977132:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300963329 00000100:00000010:2.0:1788204831.977136:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1960. 00000100:00100000:3.0:1788204831.977137:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204831.977139:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204831.977144:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204831.977148:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977152:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788204831.977155:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788204831.977158:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204831/real 1788204831] req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204831.977160:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204831.977169:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1788204831.977171:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000233989f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204831.977172:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204831.977175:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204831.977179:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204831.977183:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204831.977185:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977188:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1788204831.977192:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6296) 00000100:00000200:2.0:1788204831.977195:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204831.977206:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977211:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204831.977213:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977215:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977217:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204831.977220:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977225:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204831.977229:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977234:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204831.977237:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977238:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977239:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977241:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204831.977243:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204831.977246:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204831.977249:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204831.977251:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204831.977254:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204831.977261:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205901952:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204831.977265:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977267:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977270:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1c00 x1875062205901952/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977276:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204831.977278:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977281:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204831.977283:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204831.977285:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204831.977287:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204831.977289:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204831.977291:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204831.977294:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204831.977296:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204831.977297:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1c00. 00000100:00000001:2.0:1788204831.977300:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204831.977300:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977302:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977307:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204831.977308:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977309:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204831.977310:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204831.977311:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204831.977314:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977318:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204831.977319:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977320:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204831.977322:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204831.977323:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204831.977324:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902080:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204831.977326:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204831.977327:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204831.977328:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977330:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977332:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204831.977333:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.977336:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204831.977337:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204831.977340:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204831.977341:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902080, portal 10 00000100:00000001:2.0:1788204831.977343:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204831.977343:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:2.0:1788204831.977348:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977351:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204831.977353:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204831.977354:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204831.977356:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205902080, offset 0 00000400:00000200:2.0:1788204831.977358:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977360:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977389:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204831.977392:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204831.977394:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204831.977395:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204831.977397:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977399:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58903:58903:256:4294967295] 192.168.201.151@tcp LPNI seq info [58883:58883:8:4294967295] 00000400:00000200:2.0:1788204831.977408:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977412:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204831.977415:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977417:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204831.977420:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204831.977422:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204831.977425:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204831.977426:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204831.977428:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977440:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204831.977442:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204831.977445:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204831.977447:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204831.977448:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204831.977450:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204831.977452:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977457:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977459:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977462:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977463:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204831.977465:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204831.977468:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204831.977470:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204831.977471:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300959387 00000100:00000001:2.0:1788204831.977473:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977474:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300959387 00000100:00000001:2.0:1788204831.977476:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977477:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204831.977478:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977479:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204831.977481:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204831/real 1788204831] req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204831.977486:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204831.977486:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204831.977487:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204831.977488:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204831.977489:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204831.977491:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204831.977493:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204831.977497:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977501:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204831.977502:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977503:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977503:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204831.977505:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977509:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204831.977511:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977514:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204831.977515:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977516:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977517:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977518:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204831.977519:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204831.977521:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204831.977522:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204831.977523:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204831.977525:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204831.977530:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902080:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204831.977532:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204831.977534:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977541:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204831.977542:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977543:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204831.977545:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204831.977546:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204831.977547:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902208:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204831.977549:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204831.977550:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204831.977551:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977552:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977554:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204831.977555:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.977557:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204831.977558:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204831.977560:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204831.977561:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902208, portal 4 00000100:00000001:2.0:1788204831.977562:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204831.977563:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:2.0:1788204831.977567:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977570:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204831.977571:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204831.977572:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204831.977574:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205902208, offset 0 00000400:00000200:2.0:1788204831.977576:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977578:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977580:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204831.977581:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204831.977583:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204831.977584:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204831.977586:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977588:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58904:58904:256:4294967295] 192.168.201.151@tcp LPNI seq info [58884:58884:8:4294967295] 00000400:00000200:2.0:1788204831.977590:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977594:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204831.977597:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977599:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204831.977601:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204831.977602:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204831.977605:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204831.977606:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204831.977609:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977610:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204831.977611:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204831.977613:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204831.977615:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204831.977616:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204831.977617:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204831.977619:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977623:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977625:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977628:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977629:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204831.977631:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204831.977632:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204831.977634:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204831.977635:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300959273 00000100:00000001:2.0:1788204831.977636:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977637:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300959273 00000100:00000001:2.0:1788204831.977639:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977640:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204831.977642:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977643:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204831.977644:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204831/real 1788204831] req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204831.977649:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204831.977649:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204831.977650:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204831.977651:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204831.977652:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204831.977653:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204831.977655:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204831.977659:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977663:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204831.977663:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977664:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977665:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204831.977667:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977670:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204831.977672:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977676:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204831.977677:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977678:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977678:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977679:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204831.977680:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204831.977681:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204831.977684:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204831.977685:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204831.977687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204831.977691:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902208:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204831.977693:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204831.977694:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977698:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204831.977698:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977699:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204831.977700:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204831.977701:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204831.977703:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902336:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204831.977704:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204831.977705:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204831.977706:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204831.977707:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977709:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204831.977710:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204831.977711:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204831.977713:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204831.977714:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204831.977716:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902336, portal 4 00000100:00000001:2.0:1788204831.977717:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204831.977718:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:2.0:1788204831.977721:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977724:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204831.977726:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204831.977727:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204831.977728:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205902336, offset 0 00000400:00000200:2.0:1788204831.977730:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977732:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977734:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204831.977735:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204831.977737:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204831.977738:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204831.977739:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204831.977741:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58905:58905:256:4294967295] 192.168.201.151@tcp LPNI seq info [58885:58885:8:4294967295] 00000400:00000200:2.0:1788204831.977743:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204831.977746:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204831.977749:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977751:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204831.977753:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204831.977754:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1788204831.977757:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204831.977758:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1788204831.977760:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204831.977761:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204831.977763:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204831.977765:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204831.977766:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204831.977767:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204831.977768:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204831.977770:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977774:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977776:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977779:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977780:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204831.977782:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204831.977783:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204831.977785:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204831.977786:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300959279 00000100:00000001:2.0:1788204831.977787:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977789:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300959279 00000100:00000001:2.0:1788204831.977790:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300963329 00000100:00000001:2.0:1788204831.977791:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204831.977792:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977793:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204831.977794:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204831/real 1788204831] req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204831.977798:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204831.977799:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204831.977800:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204831.977801:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204831.977802:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204831.977802:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204831.977804:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204831.977808:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977811:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204831.977812:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977813:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977814:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204831.977816:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977820:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204831.977822:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977826:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204831.977827:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977828:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204831.977829:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977830:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204831.977831:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204831.977832:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204831.977834:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204831.977835:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204831.977837:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204831.977840:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902336:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204831.977842:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977843:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977845:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205902080/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977848:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204831.977849:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977850:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204831.977851:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204831.977852:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204831.977853:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204831.977854:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204831.977855:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204831.977856:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204831.977857:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204831.977858:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:2.0:1788204831.977860:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204831.977861:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977862:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977864:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205902208/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977867:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204831.977868:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977869:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204831.977870:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204831.977871:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204831.977871:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204831.977872:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204831.977873:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:2.0:1788204831.977874:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204831.977875:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204831.977875:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:2.0:1788204831.977877:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204831.977877:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977878:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204831.977879:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205902336/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204831.977883:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204831.977883:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204831.977884:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:2.0:1788204831.977885:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204831.977885:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204831.977886:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204831.977887:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204831.977888:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204831.977889:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204831.977889:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204831.977890:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:2.0:1788204831.977891:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204831.977892:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977893:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204831.977895:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204831.977895:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204831.977896:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204831.977898:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204832.168535:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204832.168537:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204832.168539:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168543:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204832.168544:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168546:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168547:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204832.168550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204832.168553:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204832.168554:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204832.168554:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168555:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204832.168556:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168557:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168558:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204832.168561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204832.168562:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204832.168563:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168564:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204832.168564:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168565:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204832.168566:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204832.168578:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204832.168579:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204832.168581:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204832.488475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204832.488481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204832.488482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204832.488486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204832.488488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204832.488491:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204832.488493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204832.488495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204832.488497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204832.936480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204832.936487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204832.936489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204832.936494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204832.936497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204832.936501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204832.936503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204832.936506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204832.936509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204833.000478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204833.000478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204833.000485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204833.000486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204833.000489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204833.000490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204833.000495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204833.000496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204833.000499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204833.000500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204833.000503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204833.000505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204833.000506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204833.000509:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204833.000512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204833.000512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204833.000515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204833.000518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204833.192503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204833.192513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204833.192522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204833.192524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204833.192527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204833.192529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204833.192532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204833.192534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204833.192598:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204833.192604:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204833.192608:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204833.512506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204833.512516:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204833.512518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.512524:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204833.512528:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204833.512533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204833.512536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204833.512539:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204833.512542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204833.960545:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204833.960557:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204833.960559:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204833.960566:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204833.960570:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204833.960575:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204833.960578:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204833.960581:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204833.960585:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204834.024522:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204834.024522:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204834.024534:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204834.024534:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204834.024538:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204834.024539:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204834.024546:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204834.024547:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204834.024553:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204834.024554:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204834.024557:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204834.024559:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204834.024561:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204834.024564:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204834.024568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204834.024570:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204834.024573:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204834.024576:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204834.216502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204834.216510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204834.216520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204834.216521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204834.216524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204834.216525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204834.216527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204834.216528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204834.216546:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204834.216548:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204834.216555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204834.536484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204834.536499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204834.536501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204834.536508:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204834.536512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204834.536517:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204834.536521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204834.536524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204834.536527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204834.984522:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204834.984530:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204834.984532:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204834.984536:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204834.984539:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204834.984543:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204834.984545:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204834.984548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204834.984550:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204835.048509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204835.048509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204835.048519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204835.048519:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204835.048523:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204835.048524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204835.048532:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204835.048532:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204835.048537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204835.048538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204835.048543:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204835.048544:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204835.048548:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204835.048551:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204835.048553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204835.048556:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204835.048560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204835.048562:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204835.240513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204835.240523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204835.240530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204835.240531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204835.240535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204835.240536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204835.240539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204835.240540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204835.240605:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204835.240611:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204835.240615:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204835.560441:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204835.560449:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204835.560451:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204835.560457:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204835.560461:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204835.560467:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204835.560470:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204835.560474:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204835.560477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204836.008521:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204836.008529:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204836.008530:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204836.008535:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204836.008538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204836.008542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204836.008544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204836.008546:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204836.008548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204836.072501:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204836.072501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204836.072509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204836.072510:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204836.072512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204836.072513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204836.072520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204836.072521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204836.072525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204836.072526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204836.072532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204836.072532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204836.072538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204836.072541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204836.072546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204836.072548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204836.072550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204836.072553:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204836.264502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204836.264508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204836.264513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204836.264514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204836.264517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204836.264517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204836.264520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204836.264521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204836.264536:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204836.264539:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204836.264541:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204836.584434:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300963329, at 4300963840 00000100:00000040:1.0:1788204836.584436:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300963329, at 4300963840 00000100:00000001:2.0:1788204836.584447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:0.0:1788204836.584453:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c10c0. 00000100:00000001:2.0:1788204836.584454:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204836.584458:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204836.584461:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1960. 00000100:00100000:2.0:1788204836.584465:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204836.584470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204836.584475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204836.584478:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204836.584482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204836.584486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204837.032475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204837.032483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204837.032485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204837.032490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204837.032493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204837.032497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204837.032499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204837.032502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204837.032505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204837.096481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1788204837.096481:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204837.096489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204837.096491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204837.096493:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1788204837.096496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204837.096497:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204837.096501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788204837.096503:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204837.096505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204837.096508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204837.096511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204837.096515:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204837.096516:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204837.096525:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6291 00000400:00000200:3.0:1788204837.096536:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204837.096543:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204837.096547:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204837.096552:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096555:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204837.096558:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096561:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204837.096571:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204837.096573:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204837.096576:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204837.096580:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204837.096582:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204837.096585:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204837.096588:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096590:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204837.096592:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096594:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204837.096598:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204837.096601:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096604:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204837.096607:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204837.096611:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204837.096614:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.096617:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204837.096620:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204837.096631:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3b80. 00000020:00000040:3.0:1788204837.096635:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204837.096639:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204837.096641:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.096645:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204837.096649:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204837.096653:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204837.096663:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23c00. 00000100:00000001:3.0:1788204837.096671:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204837.096683:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204837.096695:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204837.096700:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.096705:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c10db33c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204837.096731:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204837.096735:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204837.096738:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204837.096740:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204837.096742:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204837.096745:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204837.096747:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6291 00000400:00000200:3.0:1788204837.096752:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204837.096756:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204837.096759:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204837.096762:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096764:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204837.096768:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788204837.096768:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096773:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204837.096774:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204837.096777:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788204837.096778:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204837.096781:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204837.096785:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:2.0:1788204837.096786:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204837.096790:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204837.096796:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204837.096798:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204837.096798:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204837.096803:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788204837.096804:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204837.096807:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204837.096812:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.096812:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:2.0:1788204837.096818:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205902464:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204837.096818:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204837.096824:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1788204837.096825:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096829:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204837.096831:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1788204837.096832:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204837.096834:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.096837:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788204837.096838:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096842:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204837.096843:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788204837.096844:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204837.096846:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204837.096852:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204837.096853:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204837.096856:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 02000000:00000001:3.0:1788204837.096859:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204837.096861:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000100:00000001:0.0:1788204837.096864:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204837.096866:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204837.096867:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204837.096870:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00100000:0.0:1788204837.096877:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000008:00100000:3.0:1788204837.096877:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1788204837.096880:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902464, portal 25 00000100:00000001:0.0:1788204837.096888:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204837.096889:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:3.0:1788204837.096891:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.096894:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000001:0.0:1788204837.096895:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204837.096897:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204837.096904:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204837.096908:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204837.096909:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204837.096914:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1500. 00000100:00000001:0.0:1788204837.096915:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204837.096928:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:2.0:1788204837.096931:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788204837.096933:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204837.096937:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204837.096938:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204837.096941:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1788204837.096944:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 02000000:00000001:3.0:1788204837.096944:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204837.096949:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204837.096951:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205902464, offset 0 02000000:00000010:3.0:1788204837.096955:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20c00. 00000100:00000001:3.0:1788204837.096961:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204837.096963:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00100000:3.0:1788204837.096969:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204837.096976:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204837.096984:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204837.096987:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:3.0:1788204837.096989:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.096996:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bfc2cda] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204837.096997:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204837.097005:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204837.097011:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788204837.097012:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204837.097016:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:3.0:1788204837.097017:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204837.097022:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58906:58906:256:4294967295] 192.168.201.151@tcp LPNI seq info [58886:58886:8:4294967295] 00000100:00080000:3.0:1788204837.097025:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204837.097029:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1788204837.097031:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:3.0:1788204837.097033:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204837.097038:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204837.097042:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6291 00000400:00000200:3.0:1788204837.097050:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.097051:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:3.0:1788204837.097057:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204837.097061:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1788204837.097065:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:3.0:1788204837.097067:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097071:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204837.097073:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097074:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000100:00000001:3.0:1788204837.097077:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204837.097080:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204837.097082:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000800:00000200:2.0:1788204837.097085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:3.0:1788204837.097086:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204837.097089:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000200:2.0:1788204837.097091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000020:00000001:3.0:1788204837.097094:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:2.1:1788204837.097099:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:3.0:1788204837.097100:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:2.1:1788204837.097105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000100:00000040:3.0:1788204837.097108:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097113:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1788204837.097114:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097119:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000100:00000040:3.0:1788204837.097119:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097124:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204837.097127:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204837.097130:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1788204837.097130:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097136:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204837.097140:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204837.097141:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1788204837.097143:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204837.097147:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.097149:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204837.097150:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000008:00100000:3.0:1788204837.097152:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000010:2.0:1788204837.097155:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000008:00000001:3.0:1788204837.097156:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.097160:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097163:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1788204837.097164:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0700. 00000020:00000040:3.0:1788204837.097168:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000200:2.0:1788204837.097170:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204837.097172:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204837.097175:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.097179:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204837.097182:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204837.097185:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097189:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1788204837.097191:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22000. 00000100:00000040:2.0:1788204837.097195:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204837.097197:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204837.097207:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204837.097210:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097214:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204837.097220:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204837.097222:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097223:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.097227:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b25c632d] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204837.097232:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204837.097237:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300968449 00000100:00000001:3.0:1788204837.097244:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204837.097248:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1de0. 00000100:00080000:3.0:1788204837.097248:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204837.097254:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204837.097255:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204837.097257:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204837.097259:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204837.097260:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204837.097265:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6291 00000100:00000001:2.0:1788204837.097268:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300968449 00000400:00000200:3.0:1788204837.097270:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:2.0:1788204837.097275:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1000. 00000400:00000200:3.0:1788204837.097276:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204837.097281:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204837.097282:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1788204837.097286:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204837.097288:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204837.097290:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204837.097294:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.097294:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204837.097299:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788204837.097299:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204837.097304:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204837.097305:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00100000:2.0:1788204837.097308:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204837.097308:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204837.097310:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788204837.097315:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788204837.097318:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788204837.097320:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097325:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204837.097330:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204837.097335:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000001:2.0:1788204837.097338:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1788204837.097339:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000001:2.0:1788204837.097352:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204837.097355:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000100:00080000:3.0:1788204837.097359:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788204837.097361:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000001:3.0:1788204837.097387:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204837.097390:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204837.097394:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1788204837.097398:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 02000000:00000001:3.0:1788204837.097399:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.097404:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204837.097407:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:3.0:1788204837.097408:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788204837.097413:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204837.097414:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204837.097417:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204837.097424:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0a80. 00000020:00000040:3.0:1788204837.097428:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000200:2.0:1788204837.097429:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204837.097433:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204837.097436:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204837.097441:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204837.097443:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788204837.097444:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204837.097447:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788204837.097448:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097452:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:3.0:1788204837.097455:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22400. 00000100:00000001:2.0:1788204837.097456:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204837.097462:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204837.097464:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204837.097471:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204837.097480:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204837.097487:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204837.097491:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204837.097491:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204837.097498:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed2b742] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097508:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204837.097513:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097518:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204837.097518:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097520:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204837.097524:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6302) 00000100:00000001:2.0:1788204837.097525:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204837.097529:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204837.097533:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204837.097538:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204837.097542:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204837.097547:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204837.097555:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205902464:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204837.097560:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097562:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.097565:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205902464/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097572:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204837.097574:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204837.097577:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204837.097581:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204837.097583:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204837.097584:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204837.097587:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204837.097589:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:2.0:1788204837.097593:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204837.097595:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204837.097596:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:2.0:1788204837.097600:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204837.097601:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097603:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097611:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204837.097613:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097615:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204837.097617:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204837.097618:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204837.097621:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097628:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204837.097630:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.097631:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204837.097633:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204837.097635:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204837.097638:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902592:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204837.097641:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204837.097643:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204837.097644:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.097646:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204837.097649:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204837.097651:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204837.097655:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204837.097657:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204837.097661:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204837.097664:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902592, portal 10 00000100:00000001:2.0:1788204837.097666:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204837.097667:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:2.0:1788204837.097674:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097681:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204837.097683:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204837.097685:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204837.097688:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205902592, offset 0 00000400:00000200:2.0:1788204837.097691:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204837.097695:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.097699:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204837.097702:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204837.097705:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204837.097707:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204837.097710:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204837.097713:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58907:58907:256:4294967295] 192.168.201.151@tcp LPNI seq info [58887:58887:8:4294967295] 00000400:00000200:2.0:1788204837.097717:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.097723:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204837.097728:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097730:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204837.097735:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204837.097737:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204837.097741:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204837.097744:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1788204837.097747:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097749:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204837.097752:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204837.097756:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204837.097759:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204837.097761:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204837.097763:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204837.097766:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097774:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.097776:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097783:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097784:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204837.097787:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204837.097792:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204837.097795:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204837.097797:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300964507 00000100:00000001:2.0:1788204837.097800:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.097802:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300964507 00000100:00000001:2.0:1788204837.097804:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.097807:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204837.097808:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.097811:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204837.097814:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204837.097821:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204837.097822:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204837.097824:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204837.097827:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204837.097828:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204837.097830:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204837.097834:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204837.097842:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097848:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204837.097849:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097851:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.097852:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204837.097855:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097859:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204837.097861:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097865:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204837.097866:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097868:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.097869:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.097870:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204837.097872:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204837.097874:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204837.097876:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204837.097877:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204837.097879:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204837.097883:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902592:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204837.097886:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204837.097888:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097892:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204837.097892:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.097893:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204837.097895:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204837.097896:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204837.097897:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902720:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204837.097899:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204837.097901:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204837.097902:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.097903:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204837.097905:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204837.097907:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204837.097909:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204837.097911:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204837.097913:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204837.097915:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902720, portal 4 00000100:00000001:2.0:1788204837.097916:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204837.097917:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:2.0:1788204837.097921:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097925:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204837.097927:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204837.097928:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204837.097930:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205902720, offset 0 00000400:00000200:2.0:1788204837.097932:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204837.097934:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.097936:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204837.097938:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204837.097939:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204837.097941:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204837.097943:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204837.097945:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58908:58908:256:4294967295] 192.168.201.151@tcp LPNI seq info [58888:58888:8:4294967295] 00000400:00000200:2.0:1788204837.097947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.097951:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204837.097954:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097956:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204837.097958:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204837.097960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204837.097962:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204837.097978:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1788204837.097981:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.097982:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204837.097984:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204837.097986:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204837.097989:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204837.097990:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204837.097992:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204837.097994:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.097999:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.098001:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098005:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.098006:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204837.098008:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204837.098010:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204837.098012:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204837.098013:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300964393 00000100:00000001:2.0:1788204837.098015:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.098016:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300964393 00000100:00000001:2.0:1788204837.098018:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.098020:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204837.098022:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.098023:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204837.098027:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204837.098035:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204837.098036:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204837.098038:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204837.098040:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204837.098042:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204837.098044:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204837.098047:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204837.098056:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098062:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204837.098063:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098065:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.098066:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204837.098069:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098073:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204837.098075:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098079:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204837.098080:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098081:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.098081:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098082:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204837.098084:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204837.098085:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204837.098087:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204837.098088:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204837.098090:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204837.098095:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902720:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204837.098097:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204837.098099:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098102:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204837.098103:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.098105:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204837.098106:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204837.098107:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204837.098109:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902848:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204837.098111:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204837.098113:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204837.098114:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204837.098115:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204837.098118:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204837.098119:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204837.098121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204837.098123:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000400:00000010:2.0:1788204837.098124:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204837.098126:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902848, portal 4 00000100:00000001:2.0:1788204837.098128:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204837.098129:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:2.0:1788204837.098133:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098137:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204837.098138:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204837.098140:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000200:2.0:1788204837.098141:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205902848, offset 0 00000400:00000200:2.0:1788204837.098143:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204837.098145:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.098147:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204837.098149:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204837.098150:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204837.098152:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204837.098153:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204837.098155:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58909:58909:256:4294967295] 192.168.201.151@tcp LPNI seq info [58889:58889:8:4294967295] 00000400:00000200:2.0:1788204837.098158:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204837.098161:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204837.098165:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.098166:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000800:00000200:2.0:1788204837.098169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204837.098170:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1788204837.098173:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204837.098175:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1788204837.098177:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204837.098179:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:2.0:1788204837.098180:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204837.098183:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204837.098184:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:2.0:1788204837.098185:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:2.0:1788204837.098187:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204837.098189:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098193:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.098195:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098199:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.098200:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204837.098201:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204837.098203:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204837.098204:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204837.098205:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300964399 00000100:00000001:2.0:1788204837.098207:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.098208:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300964399 00000100:00000001:2.0:1788204837.098209:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300968449 00000100:00000001:2.0:1788204837.098211:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204837.098212:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.098213:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204837.098215:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204837.098219:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204837.098219:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204837.098220:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204837.098221:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204837.098222:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204837.098223:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204837.098225:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204837.098230:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098233:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204837.098234:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098235:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.098236:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204837.098238:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098241:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204837.098243:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098246:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204837.098247:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098248:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204837.098249:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098250:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204837.098251:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204837.098252:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204837.098254:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204837.098255:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204837.098257:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204837.098262:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205902848:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204837.098264:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098265:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.098267:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205902592/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098271:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204837.098272:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204837.098273:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204837.098275:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204837.098276:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204837.098276:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204837.098277:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204837.098278:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:2.0:1788204837.098280:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204837.098281:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204837.098282:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204837.098284:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204837.098285:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098286:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.098288:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205902720/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098291:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204837.098292:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204837.098293:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204837.098294:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204837.098295:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204837.098295:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204837.098296:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204837.098297:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:2.0:1788204837.098298:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204837.098299:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204837.098299:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:2.0:1788204837.098301:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204837.098301:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098302:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204837.098304:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205902848/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204837.098307:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204837.098308:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204837.098309:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204837.098311:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204837.098311:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204837.098312:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204837.098313:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204837.098314:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:2.0:1788204837.098315:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204837.098316:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204837.098316:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:2.0:1788204837.098317:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204837.098318:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098319:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.098321:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204837.098322:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204837.098323:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204837.098324:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204837.288480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204837.288488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204837.288497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204837.288499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204837.288502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204837.288503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204837.288508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204837.288509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204837.288591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204837.288595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204837.288598:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204837.608450:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204837.608460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204837.608462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204837.608468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204837.608472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204837.608476:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.608486:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204837.608490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204837.608494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204838.056469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204838.056479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204838.056482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204838.056489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204838.056492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204838.056496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204838.056498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204838.056501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204838.056504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.120487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204838.120487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204838.120498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204838.120499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204838.120503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204838.120503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.120513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204838.120513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204838.120519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204838.120520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204838.120524:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204838.120525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204838.120529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204838.120532:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204838.120534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204838.120536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204838.120540:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204838.120543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204838.312552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204838.312562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204838.312570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204838.312572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204838.312576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204838.312577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204838.312581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204838.312582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204838.312606:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204838.312609:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204838.312612:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204838.632438:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204838.632445:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204838.632447:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204838.632453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204838.632456:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204838.632459:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204838.632461:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204838.632464:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204838.632467:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204839.080452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204839.080457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204839.080459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204839.080462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204839.080465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204839.080468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204839.080469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204839.080471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204839.080472:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204839.144469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204839.144469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204839.144476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204839.144476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204839.144479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204839.144480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204839.144486:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204839.144486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204839.144491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204839.144492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204839.144496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204839.144496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204839.144501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204839.144504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204839.144506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204839.144507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204839.144509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204839.144511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204839.336479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204839.336487:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204839.336489:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204839.336493:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336496:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204839.336497:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336499:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336502:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204839.336506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204839.336509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204839.336510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204839.336578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204839.336581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204839.336583:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204839.656456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204839.656463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204839.656465:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204839.656470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204839.656474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204839.656483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204839.656486:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204839.656489:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204839.656492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204840.104494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204840.104502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204840.104504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204840.104510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204840.104513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204840.104516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204840.104519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204840.104529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204840.104532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204840.168462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204840.168462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204840.168469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204840.168470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204840.168473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204840.168474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204840.168482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204840.168482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204840.168489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204840.168489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204840.168494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204840.168494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204840.168497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204840.168500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204840.168502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204840.168503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204840.168506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204840.168509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204840.360517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204840.360521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204840.360525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204840.360526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204840.360528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204840.360529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204840.360531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204840.360532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204840.360544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204840.360546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204840.360548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204840.680430:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204840.680436:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204840.680438:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204840.680441:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204840.680444:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204840.680447:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204840.680449:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204840.680451:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204840.680452:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204841.128426:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204841.128433:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204841.128435:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204841.128441:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204841.128444:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204841.128447:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204841.128450:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204841.128459:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204841.128462:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204841.192482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204841.192490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204841.192493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204841.192494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204841.192504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204841.192505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204841.192511:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204841.192511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204841.192515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204841.192520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204841.192520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204841.192523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204841.192527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204841.192528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204841.192534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204841.192539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204841.192546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204841.192550:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204841.384451:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204841.384456:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204841.384460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204841.384462:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204841.384466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204841.384467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204841.384473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204841.384474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204841.384530:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204841.384534:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204841.384536:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204841.704494:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300968449, at 4300968960 00000100:00000040:1.0:1788204841.704494:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300968449, at 4300968960 00000100:00000010:1.0:1788204841.704511:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1000. 00000100:00000010:0.0:1788204841.704513:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1de0. 00000100:00000001:2.0:1788204841.704531:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204841.704541:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204841.704544:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204841.704549:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204841.704553:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204841.704557:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204841.704569:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204841.704572:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204841.704575:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204842.152477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204842.152483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204842.152485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204842.152489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204842.152491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204842.152495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204842.152496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204842.152498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204842.152500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204842.216464:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204842.216473:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204842.216476:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204842.216479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204842.216484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788204842.216484:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204842.216486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.216490:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:0.0:1788204842.216492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204842.216496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788204842.216498:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6297 00000100:00000001:0.0:1788204842.216500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204842.216504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204842.216506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204842.216511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204842.216511:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204842.216520:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204842.216524:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204842.216527:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216531:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204842.216533:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216536:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204842.216539:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204842.216541:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204842.216544:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204842.216547:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204842.216549:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204842.216552:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204842.216555:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216557:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204842.216558:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216560:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204842.216563:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204842.216566:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216568:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.216571:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204842.216574:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204842.216578:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.216581:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204842.216584:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204842.216594:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1180. 00000020:00000040:3.0:1788204842.216597:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204842.216601:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204842.216603:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204842.216606:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204842.216607:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204842.216610:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788204842.216611:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204842.216614:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.216616:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204842.216617:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204842.216620:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204842.216623:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204842.216624:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204842.216626:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204842.216628:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204842.216633:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21800. 00000100:00000001:3.0:1788204842.216640:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204842.216647:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204842.216655:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204842.216658:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204842.216660:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003da94af2] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204842.216680:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204842.216689:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204842.216693:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204842.216694:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204842.216696:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204842.216699:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204842.216701:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6297 00000400:00000200:3.0:1788204842.216705:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204842.216709:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204842.216711:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204842.216714:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216720:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204842.216723:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204842.216724:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204842.216725:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204842.216729:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788204842.216730:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204842.216732:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1788204842.216733:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204842.216737:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204842.216741:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000040:2.0:1788204842.216742:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204842.216746:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788204842.216751:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204842.216758:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788204842.216759:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204842.216761:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.216764:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788204842.216765:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204842.216769:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204842.216771:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204842.216771:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216776:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:2.0:1788204842.216777:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205902976:192.168.201.151@tcp:250:kworker.0 00000020:00000040:3.0:1788204842.216781:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204842.216783:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788204842.216786:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:2.0:1788204842.216787:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788204842.216789:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.216790:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.216791:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204842.216794:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788204842.216797:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1788204842.216798:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:3.0:1788204842.216802:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.216803:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204842.216811:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204842.216814:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:2.0:1788204842.216815:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa898. 00000008:00100000:3.0:1788204842.216819:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1788204842.216821:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000008:00000001:3.0:1788204842.216822:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.216826:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204842.216828:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205902976, portal 25 00000100:00000010:3.0:1788204842.216830:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204842.216831:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204842.216833:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000020:00000040:3.0:1788204842.216835:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204842.216837:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204842.216838:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.216839:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204842.216840:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204842.216841:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204842.216847:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20000. 00000100:00000040:2.0:1788204842.216849:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204842.216851:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204842.216861:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204842.216862:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204842.216868:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204842.216874:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85dd0. 00000100:00000001:3.0:1788204842.216876:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1788204842.216880:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205902976, offset 0 00000100:00000001:3.0:1788204842.216881:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204842.216889:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c42bbc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204842.216890:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204842.216901:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204842.216902:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204842.216905:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204842.216907:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:3.0:1788204842.216911:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1788204842.216914:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204842.216914:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204842.216917:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204842.216920:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:3.0:1788204842.216922:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204842.216925:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1788204842.216926:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6297 00000400:00000200:2.0:1788204842.216930:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204842.216933:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58910:58910:256:4294967295] 192.168.201.151@tcp LPNI seq info [58890:58890:8:4294967295] 00000400:00000200:2.0:1788204842.216936:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204842.216941:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204842.216948:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204842.216950:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:3.0:1788204842.216951:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204842.216957:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204842.216959:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216960:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1788204842.216964:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4000. 00000100:00000040:3.0:1788204842.216964:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216966:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204842.216968:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204842.216970:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788204842.216970:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204842.216972:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788204842.216973:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788204842.216974:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788204842.216977:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216980:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204842.216982:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216984:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204842.216986:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:2.0:1788204842.216987:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:3.0:1788204842.216988:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204842.216992:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.216993:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204842.216997:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1788204842.216997:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:3.0:1788204842.216999:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.217003:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204842.217005:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788204842.217006:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.217007:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204842.217009:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1f80. 00000020:00000040:3.0:1788204842.217011:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788204842.217013:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204842.217014:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.217015:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204842.217016:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.217017:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204842.217019:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204842.217023:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23400. 00000400:00000200:2.0:1788204842.217024:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204842.217027:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217029:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300973569 00000100:00100000:3.0:1788204842.217032:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1788204842.217037:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1f60. 00000100:00000001:3.0:1788204842.217040:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204842.217042:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217044:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1788204842.217047:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002399dd83] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204842.217051:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300973569 00000100:00000001:3.0:1788204842.217054:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204842.217055:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1d20. 00000100:00080000:3.0:1788204842.217057:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204842.217058:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788204842.217062:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204842.217064:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204842.217065:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204842.217068:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204842.217069:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217074:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.217074:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204842.217077:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6297 00000100:00000001:2.0:1788204842.217080:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1788204842.217081:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204842.217085:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204842.217086:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204842.217088:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1788204842.217088:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204842.217093:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204842.217093:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204842.217098:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204842.217099:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204842.217100:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204842.217102:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788204842.217103:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204842.217106:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1788204842.217106:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788204842.217110:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204842.217111:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788204842.217114:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204842.217115:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204842.217116:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788204842.217122:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.217128:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204842.217129:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204842.217131:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217132:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204842.217133:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204842.217136:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204842.217139:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00100000:2.0:1788204842.217140:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903104:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204842.217145:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204842.217147:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1788204842.217151:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 02000000:00000001:2.0:1788204842.217152:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204842.217154:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217159:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788204842.217159:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204842.217164:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204842.217165:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788204842.217167:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204842.217169:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204842.217173:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204842.217178:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788204842.217179:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204842.217183:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa2c0. 00000100:00000001:3.0:1788204842.217184:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204842.217188:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000010:2.0:1788204842.217190:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85908. 00000008:00000001:3.0:1788204842.217193:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204842.217195:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903104, portal 10 00000100:00000001:3.0:1788204842.217197:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217200:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204842.217202:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000010:3.0:1788204842.217204:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1880. 00000100:00000040:2.0:1788204842.217210:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204842.217210:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788204842.217216:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204842.217218:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204842.217223:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204842.217224:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788204842.217226:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1788204842.217230:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788204842.217231:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204842.217235:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85ee0. 02000000:00000010:3.0:1788204842.217237:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20800. 00000100:00000200:2.0:1788204842.217239:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205903104, offset 0 00000100:00000001:3.0:1788204842.217244:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204842.217247:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204842.217264:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788204842.217265:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204842.217269:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204842.217274:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204842.217280:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:3.0:1788204842.217284:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204842.217285:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204842.217290:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:3.0:1788204842.217291:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204842.217295:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58911:58911:255:4294967295] 192.168.201.151@tcp LPNI seq info [58891:58891:7:4294967295] 00000100:00000040:3.0:1788204842.217300:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072ac1564] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204842.217301:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204842.217315:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:3.0:1788204842.217323:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204842.217324:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204842.217330:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4c00. 00000100:00000040:3.0:1788204842.217331:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6307) 00000800:00000200:2.0:1788204842.217336:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204842.217341:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204842.217344:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204842.217347:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204842.217351:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204842.217352:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300969626 00000100:00000001:2.0:1788204842.217355:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217357:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300969626 00000100:00000001:2.0:1788204842.217358:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217360:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204842.217392:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217395:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204842.217399:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204842.217404:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204842.217406:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217407:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204842.217409:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204842.217411:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204842.217413:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903232:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204842.217415:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204842.217417:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204842.217418:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217420:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204842.217423:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204842.217425:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204842.217427:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204842.217430:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa738. 00000400:00000010:2.0:1788204842.217433:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85e58. 00000100:00000200:2.0:1788204842.217435:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903232, portal 4 00000100:00000001:2.0:1788204842.217437:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204842.217438:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:2.0:1788204842.217443:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204842.217449:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204842.217451:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204842.217453:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85770. 00000100:00000200:2.0:1788204842.217455:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205903232, offset 0 00000400:00000200:2.0:1788204842.217459:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204842.217462:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204842.217465:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204842.217467:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204842.217470:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204842.217472:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204842.217474:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204842.217476:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58912:58912:254:4294967295] 192.168.201.151@tcp LPNI seq info [58892:58892:6:4294967295] 00000400:00000200:2.0:1788204842.217480:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204842.217486:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204842.217490:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204842.217494:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4400. 00000800:00000200:2.0:1788204842.217497:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204842.217501:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204842.217505:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204842.217507:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204842.217510:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204842.217512:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300969513 00000100:00000001:2.0:1788204842.217514:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217516:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300969513 00000100:00000001:2.0:1788204842.217518:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217520:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204842.217522:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217524:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217526:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204842.217530:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204842.217531:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204842.217533:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204842.217534:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204842.217535:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204842.217539:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204842.217546:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204842.217548:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217549:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204842.217551:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204842.217554:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204842.217557:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903360:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204842.217560:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204842.217561:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204842.217562:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204842.217564:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204842.217567:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204842.217568:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204842.217571:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204842.217574:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204842.217577:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204842.217580:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903360, portal 4 00000100:00000001:2.0:1788204842.217582:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204842.217583:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:2.0:1788204842.217588:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204842.217594:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204842.217597:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204842.217599:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d85d48. 00000100:00000200:2.0:1788204842.217601:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205903360, offset 0 00000400:00000200:2.0:1788204842.217605:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204842.217609:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204842.217612:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204842.217614:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204842.217617:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204842.217619:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204842.217621:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204842.217637:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58913:58913:253:4294967295] 192.168.201.151@tcp LPNI seq info [58893:58893:5:4294967295] 00000400:00000200:2.0:1788204842.217641:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204842.217647:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204842.217651:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204842.217655:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4a00. 00000800:00000200:2.0:1788204842.217658:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204842.217662:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204842.217665:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204842.217667:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204842.217671:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204842.217673:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300969519 00000100:00000001:2.0:1788204842.217675:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217677:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300969519 00000100:00000001:2.0:1788204842.217679:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300973569 00000100:00000001:2.0:1788204842.217680:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204842.217682:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217685:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217686:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204842.217689:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204842.217691:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204842.217693:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204842.217694:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204842.217695:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204842.217697:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204842.408508:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204842.408510:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204842.408513:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408518:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204842.408520:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408524:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204842.408529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204842.408534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204842.408535:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204842.408537:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408538:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204842.408540:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408541:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408543:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204842.408548:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204842.408550:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204842.408551:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408552:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204842.408553:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408554:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204842.408555:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204842.408568:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204842.408570:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204842.408572:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204842.728482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204842.728489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204842.728491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204842.728496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204842.728499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204842.728503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204842.728506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204842.728508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204842.728511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204843.176481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204843.176487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204843.176489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204843.176494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204843.176497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204843.176500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204843.176503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204843.176505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204843.176508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204843.240485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204843.240494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204843.240496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204843.240501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204843.240505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204843.240509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204843.240512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204843.240515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204843.240518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204843.240539:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204843.240543:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204843.240545:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204843.240548:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204843.240550:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204843.240552:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204843.240554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204843.240556:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204843.240558:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204843.432457:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204843.432463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204843.432470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204843.432471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204843.432474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204843.432476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204843.432479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204843.432480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204843.432532:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204843.432540:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204843.432543:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204843.752461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204843.752469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204843.752471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.752480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204843.752484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204843.752487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204843.752491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204843.752493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204843.752496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204844.200470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204844.200477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204844.200479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204844.200485:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204844.200488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204844.200491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204844.200493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204844.200496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204844.200498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204844.264486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204844.264496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204844.264498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204844.264504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204844.264507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204844.264512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204844.264514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204844.264517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204844.264520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204844.264543:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204844.264547:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204844.264549:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204844.264552:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204844.264555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204844.264557:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204844.264559:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204844.264561:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204844.264563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204844.456514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204844.456520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204844.456525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204844.456527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204844.456529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204844.456531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204844.456534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204844.456535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204844.456552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204844.456555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204844.456557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204844.776481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204844.776488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204844.776490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204844.776495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204844.776499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204844.776502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204844.776505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204844.776507:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204844.776510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204845.224460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204845.224465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204845.224467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204845.224471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204845.224474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204845.224476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.224478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204845.224480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.224481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204845.288455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204845.288461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204845.288463:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204845.288469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204845.288472:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204845.288476:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.288478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204845.288481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.288484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204845.288620:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204845.288623:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204845.288625:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204845.288628:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204845.288630:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204845.288632:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204845.288634:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000100:1.0:1788204845.288636:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000100:00000001:0.0:1788204845.288649:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204845.288653:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1788204845.288677:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1788204845.288688:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204845.288706:0:2333:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:1.0:1788204845.288711:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204845.288714:0:2333:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1788204845.288717:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204845.288721:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4000. 00000400:00000200:1.0:1788204845.288726:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204845.288731:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788204845.288736:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85dd0 00000400:00000010:1.0:1788204845.288739:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85dd0. 00000100:00000001:1.0:1788204845.288745:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204845.288753:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.288784:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204845.288789:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.288797:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204845.288799:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788204845.288809:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204845.288811:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4c00. 00000400:00000200:1.0:1788204845.288814:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204845.288817:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788204845.288819:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85ee0 00000400:00000010:1.0:1788204845.288821:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85ee0. 00000100:00000001:1.0:1788204845.288823:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204845.288827:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.288835:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204845.288849:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.288856:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204845.288859:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788204845.288865:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204845.288867:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4400. 00000400:00000200:1.0:1788204845.288871:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204845.288875:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000100:00000001:2.0:1788204845.288877:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1788204845.288878:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85770 00000400:00000010:1.0:1788204845.288883:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85770. 00000100:00000001:2.0:1788204845.288883:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204845.288888:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204845.288888:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000200:1.0:1788204845.288895:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204845.288898:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204845] req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.288911:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204845.288918:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204845.288918:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204845.288922:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204845.288927:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa2c0. 00000100:00000001:1.0:1788204845.288930:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204845.288933:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85908 00000100:00000001:1.0:1788204845.288934:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:2.0:1788204845.288936:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85908. 00000100:00000001:2.0:1788204845.288942:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000100:1.0:1788204845.288943:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204845.288948:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4a00. 00000100:00000200:2.0:1788204845.288949:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204845.288954:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204845.288962:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000100:00000200:2.0:1788204845.288966:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204845.288968:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85d48 00000400:00000010:1.0:1788204845.288972:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85d48. 00000100:00000001:1.0:1788204845.288977:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204845.288981:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204845.288985:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1788204845.288986:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.288989:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.288992:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.289001:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1788204845.289002:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204845.289008:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204867 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289014:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204845.289021:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204845.289023:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204845.289024:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204845.289033:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204845.289036:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289038:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289040:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289042:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204845.289045:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204845.289048:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204845.289052:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204845.289055:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204845.289059:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204845.289067:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903104:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204845.289071:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204845.289074:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204845] req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204845.289081:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204845.289082:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204845.289084:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa738. 00000400:00000200:2.0:1788204845.289087:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d85e58 00000400:00000010:2.0:1788204845.289088:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d85e58. 00000100:00000001:2.0:1788204845.289091:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204845.289094:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204845.289101:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289108:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204845.289110:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289111:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289112:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204845.289116:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289122:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204845.289125:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289131:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204845.289133:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289135:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289136:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289137:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204845.289140:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204845.289142:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204845.289146:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204845.289148:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204845.289152:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204845.289160:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903232:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204845.289164:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204845.289167:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204845] req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204845.289174:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204845.289175:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204845.289177:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa898. 00000400:00000200:2.0:1788204845.289179:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204845.289181:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204845.289184:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204845.289187:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204845.289194:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289229:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204845.289230:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289232:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289233:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204845.289237:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289243:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204845.289246:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289252:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204845.289254:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289256:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289257:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289258:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204845.289260:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204845.289262:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204845.289265:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204845.289267:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204845.289270:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204845.289277:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205902976:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204845.289281:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204845.289282:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204845.289286:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205903104/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289291:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204845.289294:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204845.289297:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204845.289300:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204845.289303:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204845.289305:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204845.289307:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204845.289309:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:2.0:1788204845.289312:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204845.289314:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204845.289316:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204845.289319:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204845.289320:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289322:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204845.289325:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205903232/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289330:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204845.289331:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204845.289332:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204845.289335:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204845.289336:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204845.289337:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204845.289338:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204845.289340:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:2.0:1788204845.289342:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204845.289343:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204845.289344:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:2.0:1788204845.289346:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204845.289347:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289349:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204845.289351:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205902976/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289356:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204845.289358:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204845.289359:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204845.289361:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204845.289362:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204845.289396:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204845.289398:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204845.289399:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:2.0:1788204845.289403:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204845.289404:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204845.289406:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:2.0:1788204845.289408:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204845.289410:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289413:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204845.289415:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204845.289416:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204845.289418:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204845.289422:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204845] req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204845.289430:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204845.289432:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204845.289434:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204845.289437:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204845.289439:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204845.289441:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204845.289444:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204845.289453:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289467:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204845.289468:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289470:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289471:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204845.289475:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289486:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204845.289490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289496:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204845.289499:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289500:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204845.289502:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289504:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204845.289506:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204845.289508:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204845.289512:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204845.289515:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204845.289519:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204845.289526:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903360:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204845.289529:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289531:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204845.289534:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205903360/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204845.289540:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204845.289542:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204845.289543:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204845.289546:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204845.289548:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204845.289549:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204845.289550:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204845.289552:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:2.0:1788204845.289554:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204845.289556:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204845.289557:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:2.0:1788204845.289559:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204845.289560:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289562:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.289568:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204845.289569:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204845.289570:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204845.289571:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204845.480481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204845.480486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204845.480491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204845.480493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204845.480496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204845.480496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204845.480498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204845.480499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204845.480560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204845.480563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204845.480566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204845.800483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204845.800491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204845.800493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204845.800498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204845.800502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204845.800506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.800508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204845.800510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204845.800512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204846.248468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204846.248476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204846.248478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204846.248488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204846.248492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204846.248495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204846.248498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204846.248500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204846.248503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204846.312464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204846.312470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204846.312471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204846.312474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204846.312477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204846.312480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204846.312482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204846.312484:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204846.312486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204846.312514:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204846.312518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204846.312519:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204846.312522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204846.312525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204846.312527:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204846.312529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204846.312531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204846.312533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204846.504505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204846.504511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204846.504516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204846.504517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204846.504520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204846.504522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204846.504525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204846.504526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204846.504543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204846.504545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204846.504548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204846.824474:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300973569, at 4300974080 00000100:00000040:1.0:1788204846.824474:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300973569, at 4300974080 00000100:00000010:0.0:1788204846.824489:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1f60. 00000100:00000010:1.0:1788204846.824490:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1d20. 00000100:00000001:2.0:1788204846.824490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204846.824495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204846.824498:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204846.824501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204846.824504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204846.824508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204846.824510:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204846.824513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204846.824516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204847.272492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204847.272499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204847.272502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204847.272507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204847.272510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204847.272514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204847.272516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204847.272519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204847.272521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204847.336457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204847.336463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204847.336465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204847.336469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204847.336472:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204847.336475:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204847.336477:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204847.336480:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204847.336482:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204847.336486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204847.336491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204847.336493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204847.336496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204847.336499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204847.336502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204847.336505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204847.336507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204847.336509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204847.336516:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204847.336523:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204847.336526:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204847.336528:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204847.336532:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204847.336539:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6302 00000400:00000200:0.0:1788204847.336547:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204847.336552:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204847.336555:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204847.336558:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336561:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.336564:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336566:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204847.336569:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204847.336571:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204847.336574:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204847.336577:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204847.336580:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204847.336582:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204847.336585:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336587:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.336589:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336592:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204847.336594:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204847.336597:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336600:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204847.336602:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204847.336605:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204847.336608:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336610:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204847.336612:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204847.336618:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204847.336621:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204847.336624:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204847.336626:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336628:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204847.336630:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204847.336643:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204847.336649:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204847.336654:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204847.336661:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204847.336668:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204847.336671:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204847.336674:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204847.336690:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204847.336692:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204847.336694:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204847.336695:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204847.336696:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204847.336698:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204847.336699:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6302 00000400:00000200:0.0:1788204847.336702:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204847.336704:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204847.336706:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204847.336708:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336709:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.336711:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336712:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204847.336713:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204847.336714:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204847.336715:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204847.336716:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204847.336718:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788204847.336719:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204847.336721:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336722:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.336724:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336725:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204847.336727:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204847.336729:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336731:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204847.336732:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204847.336735:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788204847.336738:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204847.336740:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788204847.336743:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.336744:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204847.336748:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204847.336753:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204847.336757:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336759:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204847.336762:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204847.336765:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000040:2.0:1788204847.336766:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204847.336767:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204847.336770:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336774:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204847.336777:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204847.336780:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.336782:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1788204847.336786:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204847.336790:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.336791:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204847.336797:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204847.336797:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204847.336802:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204847.336807:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336812:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788204847.336816:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205903488:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204847.336817:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.336824:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788204847.336825:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204847.336829:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204847.336833:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.336839:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204847.336840:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204847.336844:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:2.0:1788204847.336846:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 00000100:00080000:0.0:1788204847.336848:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1788204847.336850:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336851:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204847.336854:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204847.336857:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204847.336859:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204847.336863:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6302 00000400:00000010:2.0:1788204847.336866:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000200:0.0:1788204847.336870:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204847.336875:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:2.0:1788204847.336877:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00080000:0.0:1788204847.336878:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204847.336884:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336887:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1788204847.336887:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903488, portal 25 00000100:00000040:0.0:1788204847.336891:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204847.336893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204847.336895:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204847.336896:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000020:00000001:0.0:1788204847.336900:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204847.336903:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204847.336906:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:2.0:1788204847.336907:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204847.336909:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204847.336914:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204847.336919:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:2.0:1788204847.336924:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204847.336927:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204847.336930:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204847.336934:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205903488, offset 0 00000100:00000040:0.0:1788204847.336940:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204847.336942:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336944:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.336948:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336950:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204847.336953:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204847.336955:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204847.336958:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204847.336962:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204847.336965:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204847.336966:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000001:0.0:1788204847.336968:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204847.336972:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.336975:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204847.336976:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00100000:0.0:1788204847.336978:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204847.336982:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204847.336984:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:0.0:1788204847.336985:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204847.336991:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000200:2.0:1788204847.336992:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788204847.336995:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204847.336999:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204847.337000:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.337002:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204847.337003:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1788204847.337005:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:0.0:1788204847.337006:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204847.337012:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000400:00000200:2.0:1788204847.337015:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58914:58914:256:4294967295] 192.168.201.151@tcp LPNI seq info [58894:58894:8:4294967295] 00000100:00000001:0.0:1788204847.337017:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204847.337026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:0.0:1788204847.337027:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204847.337045:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204847.337053:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.0:1788204847.337057:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204847.337058:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204847.337067:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1788204847.337068:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00000001:0.0:1788204847.337079:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204847.337080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204847.337085:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1788204847.337087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00080000:0.0:1788204847.337092:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204847.337094:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1788204847.337097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204847.337098:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204847.337104:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.1:1788204847.337104:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00080000:0.0:1788204847.337110:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6302 00000800:00000100:2.0:1788204847.337114:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204847.337118:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000010:2.0:1788204847.337119:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:0.0:1788204847.337125:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204847.337126:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1788204847.337130:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788204847.337136:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1788204847.337144:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204847.337149:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000001:0.0:1788204847.337151:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1788204847.337153:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000040:0.0:1788204847.337156:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204847.337160:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204847.337161:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204847.337177:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204847.337178:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204847.337180:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204847.337182:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204847.337185:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204847.337186:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204847.337198:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:2.0:1788204847.337199:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204847.337201:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204847.337206:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204847.337211:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204847.337216:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204847.337220:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204847.337222:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337222:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788204847.337225:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1788204847.337228:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204847.337230:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204847.337236:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337240:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.337241:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204847.337245:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1788204847.337246:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204847.337250:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.337252:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337253:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204847.337256:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204847.337261:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:2.0:1788204847.337261:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204847.337266:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300978689 02000000:00000001:0.0:1788204847.337267:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204847.337269:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.337270:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204847.337272:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204847.337273:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204847.337278:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000010:2.0:1788204847.337282:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c00. 00000100:00000001:0.0:1788204847.337283:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204847.337291:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204847.337293:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204847.337301:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204847.337302:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300978689 00000100:00000001:0.0:1788204847.337305:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204847.337308:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ea0. 00000100:00000040:0.0:1788204847.337312:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337314:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204847.337320:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204847.337326:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204847.337327:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337333:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788204847.337336:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6312) 00000100:00100000:2.0:1788204847.337343:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204847/real 1788204847] req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204847.337353:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204847.337356:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204847.337358:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204847.337361:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204847.337397:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204847.337400:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204847.337404:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204847.337412:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337430:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204847.337432:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337434:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.337436:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204847.337441:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337447:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204847.337452:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337459:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204847.337462:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337464:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.337465:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337468:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204847.337469:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204847.337473:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204847.337476:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204847.337479:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204847.337483:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204847.337490:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205903488:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204847.337496:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337498:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.337501:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205903488/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337506:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204847.337509:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204847.337512:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204847.337516:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204847.337518:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204847.337520:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204847.337522:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204847.337524:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:2.0:1788204847.337529:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204847.337530:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204847.337532:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204847.337535:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204847.337537:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337539:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337547:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204847.337548:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337551:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204847.337553:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204847.337554:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204847.337560:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337566:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204847.337567:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.337569:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204847.337571:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204847.337573:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204847.337576:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903616:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204847.337579:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204847.337581:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204847.337583:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.337585:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204847.337588:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204847.337590:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204847.337594:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204847.337596:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204847.337605:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204847.337608:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903616, portal 10 00000100:00000001:2.0:1788204847.337610:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204847.337611:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204847.337618:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337623:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204847.337626:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204847.337628:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204847.337630:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205903616, offset 0 00000400:00000200:2.0:1788204847.337634:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204847.337638:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.337642:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204847.337646:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204847.337648:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204847.337650:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204847.337652:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204847.337655:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58915:58915:256:4294967295] 192.168.201.151@tcp LPNI seq info [58895:58895:8:4294967295] 00000400:00000200:2.0:1788204847.337659:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.337665:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204847.337670:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.337673:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204847.337677:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204847.337679:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204847.337684:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204847.337686:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204847.337690:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.337692:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204847.337696:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204847.337699:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204847.337701:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204847.337703:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204847.337705:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204847.337708:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337715:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.337717:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337723:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337725:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204847.337728:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204847.337731:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204847.337734:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204847.337735:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300974747 00000100:00000001:2.0:1788204847.337739:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.337740:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300974747 00000100:00000001:2.0:1788204847.337742:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.337745:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204847.337747:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.337749:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204847.337752:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204847/real 1788204847] req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204847.337759:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204847.337761:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204847.337762:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204847.337764:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204847.337765:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204847.337767:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204847.337771:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204847.337780:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337787:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204847.337788:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337790:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.337791:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204847.337793:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337799:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204847.337802:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337809:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204847.337813:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337815:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.337816:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.337817:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204847.337820:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204847.337822:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204847.337825:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204847.337828:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204847.337831:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204847.337838:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903616:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204847.337843:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204847.337846:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337856:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204847.337857:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.337859:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204847.337861:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204847.337863:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204847.337865:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903744:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204847.337868:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204847.337871:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204847.337873:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.337874:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204847.337877:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204847.337879:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204847.337882:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204847.337885:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204847.337889:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204847.337891:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903744, portal 4 00000100:00000001:2.0:1788204847.337893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204847.337894:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204847.337900:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337906:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204847.337908:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204847.337910:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204847.337912:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205903744, offset 0 00000400:00000200:2.0:1788204847.337916:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204847.337920:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.337923:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204847.337926:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204847.337928:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204847.337930:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204847.337933:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204847.337936:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58916:58916:256:4294967295] 192.168.201.151@tcp LPNI seq info [58896:58896:8:4294967295] 00000400:00000200:2.0:1788204847.337940:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.337945:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204847.337950:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.337952:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204847.337955:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204847.337958:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204847.337961:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204847.337964:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204847.337966:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.337968:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204847.337971:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204847.337974:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204847.337978:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204847.337979:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204847.337981:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204847.337984:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337990:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.337993:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.337998:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.338000:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204847.338002:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204847.338005:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204847.338007:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204847.338009:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300974633 00000100:00000001:2.0:1788204847.338012:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.338013:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300974633 00000100:00000001:2.0:1788204847.338015:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.338017:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204847.338019:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.338021:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204847.338023:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204847/real 1788204847] req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204847.338029:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204847.338035:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204847.338037:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204847.338039:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204847.338041:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204847.338042:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204847.338045:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204847.338051:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338057:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204847.338058:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338060:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.338061:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204847.338064:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338069:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204847.338072:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338078:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204847.338080:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338081:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.338083:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338084:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204847.338087:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204847.338089:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204847.338092:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204847.338095:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204847.338099:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204847.338106:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903744:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204847.338109:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204847.338112:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338117:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204847.338119:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.338121:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204847.338122:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204847.338124:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204847.338126:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903872:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204847.338129:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204847.338130:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204847.338132:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204847.338133:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204847.338136:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204847.338138:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204847.338141:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204847.338143:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204847.338146:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204847.338149:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205903872, portal 4 00000100:00000001:2.0:1788204847.338151:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204847.338152:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788204847.338158:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338163:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204847.338176:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204847.338178:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204847.338182:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205903872, offset 0 00000400:00000200:2.0:1788204847.338185:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204847.338193:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.338196:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204847.338199:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204847.338202:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204847.338204:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204847.338206:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204847.338209:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58917:58917:256:4294967295] 192.168.201.151@tcp LPNI seq info [58897:58897:8:4294967295] 00000400:00000200:2.0:1788204847.338212:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204847.338217:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204847.338221:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.338224:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204847.338228:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204847.338230:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788204847.338234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204847.338236:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788204847.338239:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204847.338241:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204847.338244:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204847.338247:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204847.338249:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204847.338251:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204847.338253:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204847.338257:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338264:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.338266:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338272:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.338274:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204847.338277:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204847.338280:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204847.338283:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204847.338285:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300974639 00000100:00000001:2.0:1788204847.338287:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.338290:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300974639 00000100:00000001:2.0:1788204847.338292:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300978689 00000100:00000001:2.0:1788204847.338293:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204847.338295:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.338297:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204847.338301:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204847/real 1788204847] req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204847.338309:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204847.338310:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204847.338311:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204847.338313:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204847.338314:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204847.338315:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204847.338317:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204847.338322:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338326:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204847.338327:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338328:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.338328:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204847.338330:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338334:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204847.338336:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338340:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204847.338341:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338343:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204847.338343:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338344:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204847.338345:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204847.338347:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204847.338348:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204847.338350:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204847.338352:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204847.338356:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205903872:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204847.338358:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338359:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.338361:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205903616/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338389:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204847.338390:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204847.338391:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204847.338393:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204847.338394:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204847.338396:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204847.338398:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204847.338399:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788204847.338402:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204847.338403:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204847.338405:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204847.338407:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204847.338408:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338410:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.338413:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205903744/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338419:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204847.338420:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204847.338421:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204847.338423:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204847.338424:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204847.338425:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204847.338427:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204847.338428:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:2.0:1788204847.338430:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204847.338431:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204847.338432:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204847.338434:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204847.338435:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338437:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204847.338440:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205903872/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204872 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204847.338445:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204847.338446:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204847.338447:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204847.338448:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204847.338449:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204847.338450:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204847.338452:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204847.338453:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:2.0:1788204847.338456:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204847.338457:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204847.338458:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204847.338460:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204847.338461:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338462:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.338467:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204847.338469:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204847.338471:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204847.338473:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204847.528467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204847.528475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204847.528482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204847.528483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204847.528486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204847.528487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204847.528491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204847.528493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204847.528554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204847.528557:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204847.528560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204847.848479:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204847.848486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204847.848487:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204847.848491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788204847.848493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204847.848497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.848499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204847.848501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204847.848503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204848.296497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204848.296506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204848.296508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204848.296514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204848.296517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204848.296520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204848.296523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204848.296525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204848.296528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204848.360484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204848.360491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204848.360493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204848.360497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204848.360501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204848.360511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204848.360513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204848.360516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204848.360520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204848.360601:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204848.360604:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204848.360604:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204848.360606:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204848.360608:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204848.360609:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204848.360611:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204848.360611:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204848.360613:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204848.552496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204848.552503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204848.552508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204848.552510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204848.552513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204848.552514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204848.552516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204848.552517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204848.552533:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204848.552535:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204848.552538:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204848.872489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204848.872497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204848.872500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204848.872516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204848.872522:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204848.872526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204848.872528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204848.872530:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204848.872533:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204849.320438:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204849.320445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204849.320447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204849.320452:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204849.320456:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204849.320459:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204849.320461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204849.320464:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204849.320466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204849.384453:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204849.384459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204849.384461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204849.384466:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204849.384469:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204849.384473:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204849.384476:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204849.384478:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204849.384481:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204849.384572:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204849.384575:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204849.384576:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204849.384579:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204849.384581:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204849.384582:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204849.384584:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204849.384586:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204849.384589:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204849.576483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204849.576492:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204849.576495:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204849.576498:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576501:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204849.576503:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576505:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576507:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204849.576511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204849.576514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204849.576516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204849.576577:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204849.576580:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204849.576582:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204849.896458:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204849.896465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204849.896467:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204849.896472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204849.896475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204849.896480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204849.896482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204849.896485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204849.896487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204850.344457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204850.344463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204850.344465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204850.344470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204850.344473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204850.344477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204850.344480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204850.344482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204850.344485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204850.408473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204850.408473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204850.408481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204850.408481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204850.408486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204850.408486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204850.408494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204850.408494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204850.408500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204850.408500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204850.408506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204850.408506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204850.408508:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204850.408512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204850.408514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204850.408514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204850.408518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204850.408521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204850.600510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204850.600516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204850.600522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204850.600523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204850.600525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204850.600527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204850.600530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204850.600531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204850.600549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204850.600551:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204850.600555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204850.920485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204850.920491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204850.920493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204850.920498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204850.920501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204850.920504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204850.920508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204850.920511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204850.920515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204851.368488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204851.368496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204851.368498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204851.368503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204851.368506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204851.368510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204851.368512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204851.368515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204851.368518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204851.432448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204851.432452:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204851.432456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204851.432457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204851.432460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204851.432461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204851.432468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204851.432468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204851.432475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204851.432476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204851.432480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204851.432480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204851.432485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204851.432488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204851.432490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204851.432492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204851.432495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204851.432498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204851.624432:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204851.624439:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204851.624445:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204851.624447:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204851.624450:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204851.624452:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204851.624454:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204851.624456:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204851.624520:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204851.624524:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204851.624527:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204851.944486:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300978689, at 4300979200 00000100:00000040:1.0:1788204851.944486:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300978689, at 4300979200 00000100:00000010:1.0:1788204851.944500:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ea0. 00000100:00000010:0.0:1788204851.944502:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c00. 00000100:00000001:3.0:1788204851.944514:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204851.944519:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204851.944522:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204851.944527:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204851.944530:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204851.944534:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204851.944537:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204851.944540:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204851.944543:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204852.392485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204852.392491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204852.392493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204852.392497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204852.392499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204852.392510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204852.392514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204852.392516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204852.392519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204852.456489:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204852.456493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204852.456500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204852.456504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204852.456504:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204852.456508:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204852.456511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:3.0:1788204852.456515:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204852.456517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204852.456521:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204852.456523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204852.456527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788204852.456531:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6307 00000100:00000001:0.0:1788204852.456533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204852.456538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204852.456542:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204852.456550:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204852.456553:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204852.456557:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456560:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204852.456563:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456566:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204852.456570:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204852.456572:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204852.456575:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204852.456578:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204852.456581:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204852.456583:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204852.456586:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456588:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204852.456590:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456592:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204852.456595:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204852.456599:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456602:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204852.456606:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204852.456610:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204852.456613:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.456615:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204852.456618:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204852.456628:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:3.0:1788204852.456630:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0000. 00000100:00000001:1.0:1788204852.456633:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1788204852.456635:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 00000100:00000001:1.0:1788204852.456637:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204852.456642:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 02000000:00000001:3.0:1788204852.456642:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204852.456646:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204852.456647:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204852.456651:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788204852.456652:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788204852.456656:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788204852.456657:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204852.456660:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788204852.456662:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204852.456663:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204852.456675:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22400. 00000100:00000001:3.0:1788204852.456683:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204852.456693:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204852.456704:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204852.456708:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204852.456713:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee757f9f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204852.456739:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204852.456742:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204852.456745:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204852.456746:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204852.456748:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204852.456751:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204852.456753:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6307 00000400:00000200:3.0:1788204852.456757:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204852.456760:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204852.456763:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204852.456766:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456768:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204852.456773:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204852.456777:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1788204852.456777:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204852.456780:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788204852.456782:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:2.0:1788204852.456787:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204852.456787:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204852.456789:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204852.456793:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204852.456797:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204852.456800:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788204852.456802:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204852.456803:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.456807:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788204852.456808:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204852.456810:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204852.456814:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204852.456817:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:2.0:1788204852.456819:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205904000:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204852.456822:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204852.456825:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1788204852.456828:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:2.0:1788204852.456830:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788204852.456832:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204852.456834:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204852.456838:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204852.456839:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788204852.456844:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000010:2.0:1788204852.456847:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:3.0:1788204852.456852:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.456853:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204852.456857:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204852.456863:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788204852.456866:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1788204852.456869:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 02000000:00000001:3.0:1788204852.456870:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.456874:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:2.0:1788204852.456878:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000008:00100000:3.0:1788204852.456879:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204852.456884:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.456886:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204852.456888:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904000, portal 25 00000100:00000010:3.0:1788204852.456891:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3480. 00000100:00000001:2.0:1788204852.456893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204852.456897:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000020:00000040:3.0:1788204852.456897:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204852.456903:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204852.456906:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.456909:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204852.456910:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204852.456913:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204852.456916:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204852.456923:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22000. 00000100:00000001:3.0:1788204852.456929:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.456937:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788204852.456938:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1788204852.456945:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204852.456951:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204852.456953:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1788204852.456957:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205904000, offset 0 00000100:00000001:3.0:1788204852.456957:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204852.456966:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133b1fcc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204852.456971:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204852.456981:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204852.456982:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:3.0:1788204852.456987:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204852.456992:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:3.0:1788204852.456993:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204852.456998:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204852.457000:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204852.457001:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204852.457004:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204852.457009:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:3.0:1788204852.457010:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6307 00000400:00000200:2.0:1788204852.457016:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204852.457020:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204852.457024:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58918:58918:256:4294967295] 192.168.201.151@tcp LPNI seq info [58898:58898:8:4294967295] 00000400:00000200:2.0:1788204852.457028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204852.457034:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.457042:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:3.0:1788204852.457044:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204852.457049:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1788204852.457055:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:3.0:1788204852.457056:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457061:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204852.457063:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457065:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204852.457067:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204852.457069:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788204852.457071:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:2.0:1788204852.457072:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000020:00000040:3.0:1788204852.457075:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788204852.457093:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788204852.457099:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788204852.457103:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457108:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1788204852.457109:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:3.0:1788204852.457113:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204852.457115:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1788204852.457117:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204852.457124:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:2.1:1788204852.457129:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:3.0:1788204852.457131:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:2.1:1788204852.457136:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1788204852.457137:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204852.457144:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1788204852.457148:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788204852.457148:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1788204852.457154:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 02000000:00000001:3.0:1788204852.457154:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457158:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204852.457162:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:2.0:1788204852.457164:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:3.0:1788204852.457165:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457170:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204852.457174:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000010:3.0:1788204852.457177:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2d80. 00000400:00000200:2.0:1788204852.457180:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000020:00000040:3.0:1788204852.457182:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000010:2.0:1788204852.457184:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 02000000:00000001:3.0:1788204852.457189:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788204852.457191:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1788204852.457192:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457196:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1788204852.457197:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204852.457200:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204852.457204:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204852.457211:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20c00. 00000100:00000001:2.0:1788204852.457214:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204852.457216:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.457221:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204852.457225:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204852.457253:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457254:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204852.457256:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204852.457260:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457264:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204852.457266:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001291af88] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204852.457270:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204852.457275:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300983809 00000100:00000010:2.0:1788204852.457282:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c15a0. 00000100:00000001:3.0:1788204852.457284:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204852.457288:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204852.457291:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788204852.457295:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204852.457298:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204852.457299:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300983809 00000100:00080000:3.0:1788204852.457301:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1788204852.457302:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a20. 00000100:00000001:3.0:1788204852.457306:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204852.457307:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204852.457311:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788204852.457312:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6307 00000100:00000001:2.0:1788204852.457314:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457319:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:3.0:1788204852.457321:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:2.0:1788204852.457326:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204852/real 1788204852] req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204852.457329:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204852.457335:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204852.457340:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457342:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204852.457344:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457349:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204852.457355:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204852.457356:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000400:00000001:2.0:1788204852.457357:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788204852.457359:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:2.0:1788204852.457361:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1788204852.457385:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788204852.457388:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788204852.457391:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788204852.457395:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457398:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204852.457400:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457403:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204852.457406:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788204852.457408:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204852.457411:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204852.457413:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1788204852.457417:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 02000000:00000001:3.0:1788204852.457419:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204852.457423:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 02000000:00000001:3.0:1788204852.457424:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457428:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204852.457430:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000010:2.0:1788204852.457435:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000008:00000001:3.0:1788204852.457435:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457440:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457441:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:3.0:1788204852.457445:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3100. 00000100:00000200:2.0:1788204852.457449:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204852.457450:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788204852.457456:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204852.457458:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204852.457463:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204852.457468:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1788204852.457469:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204852.457471:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204852.457478:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23c00. 00000100:00000001:2.0:1788204852.457480:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204852.457484:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204852.457484:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457490:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.457492:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204852.457495:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788204852.457500:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204852.457513:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204852.457518:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204852.457518:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.457526:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204852.457528:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e74a292a] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457550:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204852.457551:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457555:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457558:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1788204852.457559:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6317) 00000100:00000001:2.0:1788204852.457560:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457565:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204852.457568:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204852.457571:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204852.457576:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204852.457579:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204852.457585:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204852.457593:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205904000:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204852.457597:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457599:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.457602:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205904000/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457608:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204852.457610:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204852.457613:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204852.457617:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204852.457618:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204852.457620:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204852.457622:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204852.457625:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:2.0:1788204852.457629:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204852.457631:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204852.457632:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:2.0:1788204852.457635:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204852.457636:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457637:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457644:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204852.457645:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457646:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204852.457647:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204852.457649:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204852.457651:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457656:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204852.457657:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.457658:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204852.457660:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204852.457661:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204852.457663:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904128:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204852.457665:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204852.457666:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204852.457668:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.457669:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204852.457671:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204852.457672:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204852.457675:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204852.457677:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204852.457679:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204852.457681:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904128, portal 10 00000100:00000001:2.0:1788204852.457682:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204852.457684:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:2.0:1788204852.457689:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457696:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204852.457698:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204852.457701:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204852.457703:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205904128, offset 0 00000400:00000200:2.0:1788204852.457707:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204852.457712:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.457715:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204852.457719:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204852.457721:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204852.457724:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204852.457726:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204852.457730:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58919:58919:256:4294967295] 192.168.201.151@tcp LPNI seq info [58899:58899:8:4294967295] 00000400:00000200:2.0:1788204852.457734:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.457741:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204852.457745:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.457748:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204852.457752:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204852.457755:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204852.457759:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204852.457762:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204852.457766:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.457767:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204852.457770:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204852.457773:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204852.457776:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204852.457778:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204852.457781:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204852.457784:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457791:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.457794:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457800:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457802:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204852.457805:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.457809:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204852.457812:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204852.457814:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300979867 00000100:00000001:2.0:1788204852.457816:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.457818:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300979867 00000100:00000001:2.0:1788204852.457820:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.457822:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204852.457824:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.457826:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204852.457829:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204852/real 1788204852] req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204852.457838:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204852.457839:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204852.457841:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204852.457843:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204852.457845:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204852.457847:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204852.457850:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204852.457857:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457863:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204852.457864:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457866:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.457867:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204852.457870:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457876:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204852.457880:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457886:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204852.457888:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457890:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.457891:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.457893:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204852.457895:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204852.457897:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204852.457900:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204852.457902:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204852.457906:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204852.457913:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904128:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204852.457916:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204852.457920:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457927:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204852.457928:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.457930:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204852.457932:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204852.457933:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204852.457936:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904256:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204852.457939:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204852.457940:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204852.457941:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.457942:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204852.457944:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204852.457946:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204852.457948:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204852.457950:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204852.457952:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204852.457955:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904256, portal 4 00000100:00000001:2.0:1788204852.457956:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204852.457957:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:2.0:1788204852.457961:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.457965:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204852.457967:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204852.457968:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204852.457970:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205904256, offset 0 00000400:00000200:2.0:1788204852.457972:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204852.457974:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.457976:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204852.457978:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204852.457980:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204852.457982:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204852.457983:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204852.457985:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58920:58920:256:4294967295] 192.168.201.151@tcp LPNI seq info [58900:58900:8:4294967295] 00000400:00000200:2.0:1788204852.457987:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.457991:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204852.457994:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.457995:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204852.457998:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204852.458000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204852.458002:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204852.458004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204852.458006:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.458008:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204852.458009:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204852.458012:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204852.458013:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204852.458014:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204852.458017:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204852.458019:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458023:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.458025:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458029:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.458030:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204852.458031:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.458033:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204852.458035:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204852.458036:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300979753 00000100:00000001:2.0:1788204852.458037:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.458038:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300979753 00000100:00000001:2.0:1788204852.458039:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.458041:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204852.458042:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.458043:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204852.458045:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204852/real 1788204852] req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204852.458049:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204852.458049:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204852.458050:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204852.458051:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204852.458052:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204852.458053:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204852.458055:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204852.458059:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458066:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204852.458067:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458068:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.458069:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204852.458071:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458075:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204852.458091:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458095:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204852.458096:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458097:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.458098:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458099:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204852.458100:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204852.458102:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204852.458104:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204852.458105:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204852.458108:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204852.458115:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904256:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204852.458118:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204852.458121:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458127:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204852.458128:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.458131:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204852.458133:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204852.458135:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204852.458137:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904384:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204852.458140:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204852.458142:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204852.458143:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204852.458145:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204852.458148:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204852.458149:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204852.458153:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204852.458155:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204852.458158:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204852.458161:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904384, portal 4 00000100:00000001:2.0:1788204852.458162:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204852.458163:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:2.0:1788204852.458167:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458171:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204852.458172:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204852.458173:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204852.458175:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205904384, offset 0 00000400:00000200:2.0:1788204852.458177:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204852.458180:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.458182:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204852.458184:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204852.458185:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204852.458187:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204852.458188:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204852.458190:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58921:58921:256:4294967295] 192.168.201.151@tcp LPNI seq info [58901:58901:8:4294967295] 00000400:00000200:2.0:1788204852.458193:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204852.458197:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204852.458201:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.458204:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204852.458207:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204852.458210:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204852.458213:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204852.458216:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204852.458219:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204852.458221:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204852.458224:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204852.458227:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204852.458230:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204852.458232:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204852.458234:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204852.458237:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458244:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.458247:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458251:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.458252:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204852.458255:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204852.458256:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204852.458258:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204852.458260:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300979759 00000100:00000001:2.0:1788204852.458261:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.458263:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300979759 00000100:00000001:2.0:1788204852.458264:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300983809 00000100:00000001:2.0:1788204852.458265:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204852.458266:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.458267:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204852.458269:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204852/real 1788204852] req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204852.458274:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204852.458275:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204852.458276:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204852.458277:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204852.458278:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204852.458279:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204852.458281:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204852.458286:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458289:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204852.458289:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458290:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.458291:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204852.458293:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458296:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204852.458298:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458303:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204852.458304:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458305:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204852.458306:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458307:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204852.458308:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204852.458309:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204852.458311:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204852.458312:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204852.458314:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204852.458319:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904384:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204852.458321:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458322:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.458323:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205904128/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458327:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204852.458327:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204852.458328:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204852.458330:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204852.458331:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204852.458331:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204852.458332:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204852.458333:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:2.0:1788204852.458335:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204852.458335:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204852.458336:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:2.0:1788204852.458338:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204852.458339:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458340:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.458341:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205904256/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458345:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204852.458345:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204852.458346:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204852.458347:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204852.458348:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204852.458349:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204852.458350:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204852.458350:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:2.0:1788204852.458351:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204852.458353:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204852.458353:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:2.0:1788204852.458355:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204852.458356:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458356:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204852.458358:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205904384/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204852.458362:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204852.458381:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204852.458384:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204852.458386:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204852.458387:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204852.458388:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204852.458389:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204852.458391:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:2.0:1788204852.458393:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204852.458394:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204852.458396:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:2.0:1788204852.458398:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204852.458399:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458401:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204852.458404:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204852.458406:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204852.458407:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204852.458409:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204852.648513:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204852.648517:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204852.648519:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648522:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204852.648524:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648526:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648527:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204852.648531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204852.648533:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204852.648534:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204852.648535:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648536:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204852.648537:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648538:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648539:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204852.648542:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204852.648543:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204852.648544:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648544:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204852.648545:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648546:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204852.648547:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204852.648562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204852.648564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204852.648566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204852.968556:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204852.968565:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204852.968568:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204852.968576:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204852.968582:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204852.968586:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204852.968589:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204852.968592:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204852.968595:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204853.416453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204853.416457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204853.416459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204853.416462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204853.416464:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204853.416467:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204853.416469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204853.416470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204853.416472:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204853.480476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204853.480483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204853.480486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204853.480489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204853.480492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204853.480493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204853.480497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204853.480498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204853.480502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204853.480504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204853.480508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204853.480508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204853.480512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204853.480513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204853.480519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204853.480520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204853.480524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204853.480526:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204853.672447:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204853.672452:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204853.672456:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204853.672457:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204853.672460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204853.672461:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204853.672465:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204853.672466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204853.672525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204853.672529:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204853.672532:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204853.992476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204853.992482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204853.992484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204853.992489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204853.992493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204853.992497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204853.992500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204853.992503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204853.992506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204854.440449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204854.440456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204854.440458:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204854.440462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204854.440465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204854.440467:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204854.440469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204854.440470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204854.440473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.504461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204854.504467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204854.504468:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204854.504472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204854.504473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204854.504476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204854.504481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204854.504487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204854.504488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204854.504493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204854.504493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204854.504496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204854.504498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204854.504502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204854.504505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.504506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204854.504509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204854.504513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204854.696494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204854.696498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204854.696502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204854.696503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204854.696505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204854.696506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204854.696508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204854.696508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204854.696521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204854.696522:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204854.696524:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204855.016464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204855.016470:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204855.016471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204855.016475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204855.016477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204855.016480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204855.016482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204855.016484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204855.016486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204855.464455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204855.464460:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204855.464462:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204855.464466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204855.464468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204855.464471:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204855.464473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204855.464474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204855.464476:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204855.528450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204855.528450:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204855.528456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204855.528457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204855.528460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204855.528463:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204855.528466:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204855.528467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204855.528471:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204855.528472:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204855.528475:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204855.528476:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204855.528478:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204855.528481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204855.528483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204855.528489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204855.528492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204855.528495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204855.720457:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204855.720464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204855.720470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204855.720472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204855.720475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204855.720476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204855.720480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204855.720481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204855.720541:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204855.720545:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204855.720549:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204856.040464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204856.040470:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204856.040471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204856.040475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204856.040477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204856.040480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204856.040482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204856.040485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204856.040487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204856.488438:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204856.488444:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204856.488445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204856.488449:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204856.488451:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204856.488454:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204856.488455:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204856.488457:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204856.488459:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204856.552457:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204856.552457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204856.552464:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204856.552464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204856.552467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204856.552468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204856.552474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788204856.552474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204856.552479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204856.552480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204856.552485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204856.552485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204856.552490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204856.552492:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204856.552495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204856.552495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204856.552498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204856.552500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204856.744523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204856.744529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204856.744533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204856.744533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204856.744536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204856.744536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204856.744538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204856.744539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204856.744553:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204856.744555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204856.744556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204857.064453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204857.064458:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300983809, at 4300984320 00000100:00000001:3.0:1788204857.064460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204857.064462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204857.064469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204857.064473:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c15a0. 00000100:00000001:3.0:1788204857.064474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788204857.064477:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300983809, at 4300984320 00000100:00000001:3.0:1788204857.064480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204857.064485:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1788204857.064487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a20. 00000100:00000001:3.0:1788204857.064488:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204857.064494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.512460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204857.512465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204857.512467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204857.512470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204857.512473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204857.512475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204857.512477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204857.512479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204857.512481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204857.576445:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204857.576447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204857.576453:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204857.576456:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204857.576456:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204857.576459:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204857.576462:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:1.0:1788204857.576463:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204857.576469:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204857.576469:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204857.576474:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6312 00000100:00000001:0.0:1788204857.576475:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204857.576480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204857.576483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204857.576485:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204857.576487:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204857.576494:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204857.576499:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204857.576503:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576506:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.576509:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576512:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204857.576516:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204857.576518:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:0.0:1788204857.576521:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000020:00000001:1.0:1788204857.576521:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788204857.576526:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1788204857.576527:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:0.0:1788204857.576530:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204857.576531:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204857.576536:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00100000:0.0:1788204857.576537:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000040:1.0:1788204857.576539:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204857.576543:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204857.576543:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.576545:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576547:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788204857.576548:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788204857.576550:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00000001:0.0:1788204857.576553:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788204857.576554:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204857.576556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204857.576558:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204857.576560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204857.576561:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204857.576565:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204857.576568:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576569:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204857.576571:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204857.576579:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788204857.576582:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204857.576585:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204857.576586:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576588:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204857.576590:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204857.576594:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204857.576599:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788204857.576604:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204857.576610:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204857.576617:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204857.576622:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204857.576626:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204857.576654:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204857.576657:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204857.576660:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204857.576662:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204857.576663:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204857.576665:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204857.576667:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6312 00000400:00000200:1.0:1788204857.576670:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204857.576673:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204857.576675:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204857.576678:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576679:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.576681:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576683:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204857.576684:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204857.576685:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204857.576686:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204857.576688:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204857.576688:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788204857.576692:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204857.576692:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788204857.576694:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204857.576695:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204857.576698:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576700:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204857.576703:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204857.576704:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576707:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204857.576711:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204857.576715:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576719:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204857.576722:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204857.576726:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788204857.576727:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204857.576729:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204857.576730:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576733:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204857.576735:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1788204857.576737:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204857.576740:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.576740:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204857.576741:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204857.576746:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 02000000:00000001:2.0:1788204857.576747:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204857.576749:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204857.576755:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788204857.576755:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205904512:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788204857.576758:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576762:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204857.576764:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204857.576765:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204857.576768:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.576769:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204857.576773:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204857.576774:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 02000000:00000001:2.0:1788204857.576776:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788204857.576780:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204857.576784:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 00000100:00100000:1.0:1788204857.576789:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204857.576789:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576799:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204857.576799:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204857.576802:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204857.576807:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204857.576807:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000001:1.0:1788204857.576816:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204857.576817:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00080000:1.0:1788204857.576818:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204857.576823:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204857.576823:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204857.576824:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1788204857.576826:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904512, portal 25 00000100:00000001:1.0:1788204857.576827:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204857.576831:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6312 00000100:00000001:2.0:1788204857.576831:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204857.576834:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 00000400:00000200:1.0:1788204857.576837:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204857.576843:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:2.0:1788204857.576844:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204857.576845:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204857.576850:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576854:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.576857:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576861:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204857.576862:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788204857.576863:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204857.576865:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204857.576867:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:2.0:1788204857.576868:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:1.0:1788204857.576869:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204857.576871:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000010:2.0:1788204857.576871:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000020:00000001:1.0:1788204857.576875:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:2.0:1788204857.576877:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205904512, offset 0 00000100:00000040:1.0:1788204857.576879:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576882:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.576884:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576887:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204857.576888:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000040:1.0:1788204857.576890:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204857.576894:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576895:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204857.576896:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204857.576898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788204857.576899:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204857.576902:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.576904:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204857.576905:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00100000:1.0:1788204857.576907:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204857.576910:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.576911:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204857.576912:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204857.576917:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000400:00000200:2.0:1788204857.576919:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:1.0:1788204857.576920:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204857.576923:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1788204857.576924:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1788204857.576925:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.576928:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.576929:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1788204857.576933:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58922:58922:256:4294967295] 192.168.201.151@tcp LPNI seq info [58902:58902:8:4294967295] 02000000:00000001:1.0:1788204857.576934:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204857.576937:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.576941:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:1.0:1788204857.576945:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000400:00000200:2.0:1788204857.576951:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788204857.576952:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204857.576957:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00100000:1.0:1788204857.576961:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1788204857.576962:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204857.576971:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204857.576975:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.0:1788204857.576976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1788204857.576981:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204857.576985:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788204857.576999:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204857.577005:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204857.577007:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1788204857.577009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00080000:1.0:1788204857.577011:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204857.577012:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204857.577013:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204857.577015:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204857.577016:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6312 00000400:00000200:1.0:1788204857.577019:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204857.577021:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204857.577022:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204857.577024:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.577025:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.577028:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.577031:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204857.577034:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1788204857.577035:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000040:1.0:1788204857.577036:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204857.577039:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:2.0:1788204857.577040:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000020:00000040:1.0:1788204857.577041:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204857.577044:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204857.577045:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1788204857.577046:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204857.577051:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577054:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788204857.577055:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204857.577059:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204857.577060:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204857.577061:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000020:00000040:1.0:1788204857.577062:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204857.577066:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:2.0:1788204857.577066:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:1.0:1788204857.577071:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204857.577072:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204857.577073:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1788204857.577074:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204857.577078:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204857.577079:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204857.577080:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204857.577082:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204857.577087:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.577090:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204857.577094:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204857.577096:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204857.577097:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788204857.577098:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204857.577100:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204857.577102:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204857.577104:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1788204857.577105:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204857.577106:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204857.577110:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be400. 00000100:00000001:1.0:1788204857.577113:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204857.577120:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204857.577122:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577127:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204857.577131:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204857.577133:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577135:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204857.577137:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204857.577144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204857.577149:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300988929 00000100:00000001:1.0:1788204857.577154:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204857.577156:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1cc0. 00000100:00000040:1.0:1788204857.577161:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6322) 00000100:00000001:2.0:1788204857.577162:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204857.577169:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300988929 00000100:00000010:2.0:1788204857.577171:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c17e0. 00000100:00000001:2.0:1788204857.577174:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204857.577178:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204857.577180:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577181:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204857.577185:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204857/real 1788204857] req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204857.577190:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204857.577192:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204857.577194:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204857.577195:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204857.577196:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204857.577198:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204857.577200:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204857.577205:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577209:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204857.577210:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577211:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577212:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204857.577214:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577218:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204857.577221:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577225:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204857.577227:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577228:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577229:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577230:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204857.577231:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204857.577233:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204857.577235:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204857.577236:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204857.577239:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204857.577243:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205904512:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204857.577248:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577250:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577251:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205904512/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577255:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204857.577256:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577258:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204857.577260:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204857.577261:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204857.577262:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204857.577263:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204857.577265:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:2.0:1788204857.577267:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204857.577268:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204857.577269:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000001:2.0:1788204857.577271:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204857.577272:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577273:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577278:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204857.577279:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577281:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204857.577281:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204857.577282:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204857.577284:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577289:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204857.577290:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577291:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204857.577292:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204857.577293:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204857.577295:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904640:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204857.577296:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204857.577298:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204857.577298:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577300:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577301:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204857.577303:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.577305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204857.577306:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204857.577308:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204857.577310:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904640, portal 10 00000100:00000001:2.0:1788204857.577311:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204857.577312:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:2.0:1788204857.577316:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577320:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204857.577321:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204857.577323:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204857.577324:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205904640, offset 0 00000400:00000200:2.0:1788204857.577326:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577329:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577331:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204857.577340:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204857.577342:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204857.577344:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204857.577345:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577347:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58923:58923:256:4294967295] 192.168.201.151@tcp LPNI seq info [58903:58903:8:4294967295] 00000400:00000200:2.0:1788204857.577350:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577354:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204857.577356:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577358:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204857.577361:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204857.577379:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204857.577383:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204857.577385:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204857.577387:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577388:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204857.577390:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204857.577392:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204857.577394:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204857.577395:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204857.577397:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204857.577399:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577404:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577406:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577409:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577410:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204857.577412:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204857.577415:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204857.577417:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204857.577418:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300984987 00000100:00000001:2.0:1788204857.577419:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577421:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300984987 00000100:00000001:2.0:1788204857.577422:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577423:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204857.577424:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577425:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204857.577427:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204857/real 1788204857] req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204857.577432:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204857.577433:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204857.577434:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204857.577435:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204857.577436:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204857.577437:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204857.577439:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204857.577444:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577447:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204857.577448:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577449:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577450:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204857.577452:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577455:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204857.577457:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577460:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204857.577462:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577463:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577463:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577464:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204857.577466:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204857.577467:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204857.577468:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204857.577470:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204857.577472:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204857.577476:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904640:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204857.577478:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204857.577480:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577488:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204857.577488:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577490:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204857.577491:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204857.577492:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204857.577494:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904768:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204857.577495:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204857.577496:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204857.577497:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577498:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577500:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204857.577501:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.577503:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204857.577504:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204857.577506:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204857.577507:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904768, portal 4 00000100:00000001:2.0:1788204857.577508:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204857.577509:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 00000100:00000040:2.0:1788204857.577512:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577516:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204857.577517:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204857.577518:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204857.577520:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205904768, offset 0 00000400:00000200:2.0:1788204857.577522:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577524:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577526:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204857.577528:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204857.577529:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204857.577531:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204857.577532:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577534:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58924:58924:256:4294967295] 192.168.201.151@tcp LPNI seq info [58904:58904:8:4294967295] 00000400:00000200:2.0:1788204857.577537:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577540:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204857.577543:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577545:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204857.577547:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204857.577549:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204857.577552:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204857.577553:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204857.577555:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577557:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204857.577558:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204857.577560:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204857.577562:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204857.577563:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204857.577564:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204857.577566:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577570:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577572:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577575:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577576:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204857.577578:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204857.577579:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204857.577581:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204857.577583:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300984873 00000100:00000001:2.0:1788204857.577585:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577587:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300984873 00000100:00000001:2.0:1788204857.577589:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577591:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204857.577593:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577594:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204857.577598:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204857/real 1788204857] req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204857.577605:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204857.577614:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204857.577616:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204857.577619:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204857.577620:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204857.577622:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204857.577626:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204857.577633:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577637:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204857.577638:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577639:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577640:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204857.577642:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577646:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204857.577648:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577652:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204857.577654:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577654:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577655:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577656:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204857.577657:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204857.577659:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204857.577661:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204857.577662:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204857.577664:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204857.577669:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904768:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204857.577671:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204857.577672:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577676:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204857.577676:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577677:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204857.577678:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204857.577680:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204857.577681:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904896:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204857.577683:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204857.577684:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204857.577685:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204857.577686:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577688:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204857.577689:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204857.577691:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204857.577692:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204857.577694:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204857.577696:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205904896, portal 4 00000100:00000001:2.0:1788204857.577697:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204857.577698:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000040:2.0:1788204857.577701:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577705:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204857.577707:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204857.577708:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204857.577710:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205904896, offset 0 00000400:00000200:2.0:1788204857.577712:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577714:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577716:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204857.577718:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204857.577720:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204857.577721:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204857.577722:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204857.577724:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58925:58925:256:4294967295] 192.168.201.151@tcp LPNI seq info [58905:58905:8:4294967295] 00000400:00000200:2.0:1788204857.577727:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204857.577730:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204857.577733:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577735:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204857.577737:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204857.577739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1788204857.577742:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204857.577743:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1788204857.577745:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204857.577747:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204857.577748:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204857.577750:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204857.577752:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204857.577753:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204857.577754:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204857.577756:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577760:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577762:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577766:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577767:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204857.577769:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204857.577771:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204857.577772:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204857.577774:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300984879 00000100:00000001:2.0:1788204857.577775:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577776:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300984879 00000100:00000001:2.0:1788204857.577778:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300988929 00000100:00000001:2.0:1788204857.577779:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204857.577780:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577781:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204857.577782:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204857/real 1788204857] req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204857.577787:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204857.577787:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204857.577788:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204857.577790:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204857.577790:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204857.577791:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204857.577793:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204857.577797:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577800:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204857.577801:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577802:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577803:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204857.577804:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577808:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204857.577810:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577814:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204857.577815:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577816:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204857.577817:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577818:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204857.577819:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204857.577820:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204857.577822:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204857.577823:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204857.577825:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204857.577828:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205904896:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204857.577830:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577831:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577833:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205904640/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577837:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204857.577838:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577839:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204857.577840:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204857.577841:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204857.577842:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204857.577843:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204857.577844:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:2.0:1788204857.577846:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204857.577846:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204857.577847:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:2.0:1788204857.577849:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204857.577850:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577851:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577853:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205904768/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577856:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204857.577857:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577858:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204857.577859:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204857.577860:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204857.577860:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204857.577861:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204857.577862:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:2.0:1788204857.577863:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204857.577864:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204857.577864:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:2.0:1788204857.577866:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204857.577866:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577867:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204857.577868:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205904896/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204857.577872:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204857.577872:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204857.577873:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204857.577874:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204857.577875:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204857.577875:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204857.577876:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204857.577877:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be400. 02000000:00000001:2.0:1788204857.577878:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204857.577878:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204857.577879:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00000001:2.0:1788204857.577880:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204857.577881:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577881:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204857.577884:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204857.577884:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204857.577886:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204857.577887:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204857.768469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204857.768476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204857.768481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204857.768483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204857.768487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204857.768488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204857.768491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204857.768492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204857.768539:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204857.768543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204857.768546:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204858.088450:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204858.088456:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204858.088457:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204858.088462:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204858.088465:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204858.088467:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204858.088469:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204858.088471:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204858.088473:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204858.536497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204858.536505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204858.536507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204858.536513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204858.536518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204858.536522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204858.536525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204858.536536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204858.536539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204858.600499:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204858.600506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204858.600508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204858.600514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204858.600514:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204858.600520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204858.600520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204858.600524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204858.600527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204858.600530:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204858.600531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204858.600535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204858.600536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204858.600539:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204858.600542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204858.600543:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204858.600547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204858.600549:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204858.792540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204858.792547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204858.792552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204858.792553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204858.792556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204858.792557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204858.792561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204858.792562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204858.792586:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204858.792589:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204858.792593:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204859.112472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204859.112479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204859.112481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204859.112486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204859.112490:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204859.112494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204859.112497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204859.112499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204859.112502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204859.560507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204859.560517:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204859.560519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204859.560526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204859.560531:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204859.560536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204859.560539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204859.560543:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204859.560547:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.624446:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204859.624449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204859.624455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204859.624457:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204859.624458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204859.624461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.624466:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204859.624467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204859.624473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204859.624473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204859.624478:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204859.624479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204859.624482:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204859.624485:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204859.624488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204859.624488:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204859.624491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204859.624495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204859.816502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204859.816513:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204859.816516:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204859.816521:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816524:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204859.816527:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816528:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816531:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204859.816542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204859.816546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204859.816547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204859.816602:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204859.816607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204859.816611:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204860.136449:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204860.136456:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204860.136458:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204860.136463:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204860.136466:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204860.136470:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204860.136472:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204860.136474:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204860.136477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204860.584487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204860.584495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204860.584497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204860.584502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204860.584505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204860.584509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204860.584511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204860.584513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204860.584516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204860.648477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204860.648477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204860.648487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204860.648487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204860.648491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204860.648491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204860.648500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204860.648500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204860.648508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204860.648508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204860.648515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204860.648515:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204860.648519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204860.648523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204860.648526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204860.648528:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204860.648531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204860.648534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204860.840486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204860.840493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204860.840499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204860.840500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204860.840502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204860.840503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204860.840505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204860.840506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204860.840521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204860.840523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204860.840525:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204861.160504:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204861.160512:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204861.160514:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204861.160525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204861.160528:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204861.160532:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204861.160535:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204861.160538:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204861.160540:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204861.608515:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204861.608523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204861.608525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204861.608530:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204861.608533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204861.608536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204861.608538:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204861.608540:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204861.608542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204861.672458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204861.672458:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204861.672467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204861.672468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204861.672472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204861.672472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204861.672479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204861.672480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204861.672485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204861.672485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204861.672489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204861.672490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204861.672492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204861.672495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204861.672498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204861.672507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204861.672511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204861.672514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204861.864486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204861.864495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204861.864502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204861.864504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204861.864508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204861.864510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204861.864514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204861.864515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204861.864588:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204861.864592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204861.864596:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204862.184484:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300988929, at 4300989440 00000100:00000040:0.0:1788204862.184485:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300988929, at 4300989440 00000100:00000001:3.0:1788204862.184485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204862.184494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204862.184499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204862.184504:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204862.184507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204862.184511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204862.184514:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000001:3.0:1788204862.184515:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1788204862.184517:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000001:3.0:1788204862.184520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204862.184526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.632468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204862.632475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204862.632477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204862.632483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204862.632486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204862.632491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204862.632494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204862.632497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204862.632501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204862.696451:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204862.696453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204862.696454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204862.696462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204862.696462:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204862.696464:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204862.696467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.696467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696469:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204862.696476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204862.696476:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204862.696476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204862.696484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204862.696484:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204862.696489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204862.696490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.696493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204862.696495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788204862.696496:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6317 00000100:00000001:0.0:1788204862.696501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.696506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204862.696507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204862.696509:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204862.696510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.696515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204862.696526:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204862.696531:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204862.696534:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696537:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.696539:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696540:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204862.696544:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204862.696545:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204862.696547:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204862.696551:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204862.696552:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204862.696554:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204862.696556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696557:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.696558:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204862.696562:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204862.696564:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696566:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204862.696568:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204862.696571:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204862.696573:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696575:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204862.696577:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204862.696586:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788204862.696588:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204862.696592:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204862.696594:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696596:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204862.696599:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204862.696603:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204862.696611:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:1.0:1788204862.696618:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204862.696626:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204862.696634:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204862.696638:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204862.696643:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204862.696664:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204862.696667:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204862.696670:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204862.696672:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204862.696674:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204862.696677:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204862.696680:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6317 00000400:00000200:1.0:1788204862.696685:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204862.696689:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204862.696690:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204862.696692:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788204862.696693:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696697:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204862.696698:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788204862.696699:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696702:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204862.696703:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1788204862.696704:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204862.696707:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204862.696708:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204862.696710:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204862.696712:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204862.696713:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204862.696714:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204862.696720:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696723:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.696726:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696728:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204862.696729:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204862.696730:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696731:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204862.696732:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204862.696739:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204862.696739:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788204862.696742:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.696742:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696747:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204862.696749:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1788204862.696751:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204862.696754:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1788204862.696755:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696760:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.696760:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204862.696763:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000100:00100000:2.0:1788204862.696767:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205905024:192.168.201.151@tcp:250:kworker.0 00000020:00000040:1.0:1788204862.696768:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204862.696772:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204862.696775:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.696776:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788204862.696777:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204862.696780:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204862.696782:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.696783:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:1.0:1788204862.696785:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 02000000:00000001:2.0:1788204862.696787:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696789:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.696792:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1788204862.696795:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1788204862.696800:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204862.696802:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696808:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204862.696810:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204862.696811:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204862.696815:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000040:1.0:1788204862.696816:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204862.696822:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000001:1.0:1788204862.696824:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204862.696828:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204862.696830:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:2.0:1788204862.696830:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905024, portal 25 00000100:00000001:1.0:1788204862.696834:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204862.696835:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788204862.696836:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204862.696837:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000001:1.0:1788204862.696839:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204862.696842:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6317 00000100:00000040:2.0:1788204862.696845:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204862.696848:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204862.696854:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204862.696858:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788204862.696859:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204862.696864:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788204862.696868:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:2.0:1788204862.696868:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204862.696872:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205905024, offset 0 00000100:00000001:1.0:1788204862.696873:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.696877:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696882:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204862.696883:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000001:1.0:1788204862.696887:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204862.696889:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204862.696891:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204862.696894:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:2.0:1788204862.696897:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788204862.696898:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204862.696913:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204862.696919:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:1.0:1788204862.696920:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204862.696926:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204862.696931:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.696936:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204862.696936:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788204862.696939:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204862.696942:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204862.696943:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1788204862.696947:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788204862.696949:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696952:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204862.696956:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204862.696957:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58926:58926:256:4294967295] 192.168.201.151@tcp LPNI seq info [58906:58906:8:4294967295] 02000000:00000001:1.0:1788204862.696961:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204862.696963:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788204862.696967:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.696971:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204862.696976:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:2.0:1788204862.696976:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000008:00000001:1.0:1788204862.696983:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204862.696991:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204862.696994:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204862.696997:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00000010:1.0:1788204862.697004:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000800:00000200:2.0:1788204862.697005:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204862.697010:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204862.697017:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:1.0:1788204862.697019:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.1:1788204862.697022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 02000000:00000001:1.0:1788204862.697028:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:2.0:1788204862.697030:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788204862.697031:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204862.697034:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000100:00000001:1.0:1788204862.697035:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204862.697040:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204862.697043:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204862.697045:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:1.0:1788204862.697048:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000400:00000200:2.0:1788204862.697053:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788204862.697056:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204862.697059:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204862.697062:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00100000:1.0:1788204862.697065:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204862.697070:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204862.697077:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204862.697078:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204862.697083:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204862.697092:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697095:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.697102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204862.697106:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204862.697112:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204862.697114:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697117:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1788204862.697120:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204862.697123:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204862.697125:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204862.697128:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204862.697140:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6317 00000100:00000001:2.0:1788204862.697143:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204862.697147:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697151:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204862.697154:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204862.697157:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300994049 00000100:00080000:1.0:1788204862.697159:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000010:2.0:1788204862.697163:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1420. 00000100:00000040:1.0:1788204862.697164:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.697169:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204862.697172:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204862.697172:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788204862.697177:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204862.697180:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204862.697180:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300994049 00000020:00000040:1.0:1788204862.697184:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204862.697187:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204862.697189:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000010:2.0:1788204862.697191:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c60. 00000020:00000001:1.0:1788204862.697192:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204862.697196:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1788204862.697197:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204862.697203:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788204862.697204:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.697209:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204862.697209:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697214:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788204862.697215:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204862.697219:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:2.0:1788204862.697223:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204862/real 1788204862] req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204862.697224:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204862.697231:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204862.697237:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204862.697238:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1788204862.697241:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1788204862.697241:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204862.697245:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 02000000:00000001:1.0:1788204862.697247:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788204862.697249:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 02000000:00000001:1.0:1788204862.697252:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204862.697253:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:1.0:1788204862.697256:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204862.697258:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:1.0:1788204862.697260:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204862.697264:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.697265:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204862.697271:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000100:00000200:2.0:1788204862.697272:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204862.697276:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204862.697282:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204862.697284:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204862.697288:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204862.697289:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204862.697293:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204862.697293:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788204862.697299:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:2.0:1788204862.697306:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204862.697307:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697309:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697314:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:1.0:1788204862.697315:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204862.697317:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204862.697326:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204862.697328:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204862.697333:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204862.697340:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697341:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204862.697348:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204862.697357:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697358:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204862.697362:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697380:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.697381:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697383:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204862.697385:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204862.697388:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1788204862.697389:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6327) 00000100:00080000:2.0:1788204862.697392:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204862.697395:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204862.697398:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204862.697404:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205905024:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204862.697408:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697409:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.697411:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205905024/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697415:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204862.697418:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204862.697420:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204862.697423:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204862.697424:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204862.697426:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204862.697428:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204862.697431:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:2.0:1788204862.697434:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204862.697437:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204862.697438:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00000001:2.0:1788204862.697441:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204862.697443:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697445:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697454:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204862.697455:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697457:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204862.697459:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204862.697460:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204862.697463:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697470:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204862.697471:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697473:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204862.697474:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204862.697476:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204862.697479:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905152:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204862.697482:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204862.697484:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204862.697486:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697487:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204862.697490:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204862.697493:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204862.697497:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204862.697499:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204862.697503:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204862.697506:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905152, portal 10 00000100:00000001:2.0:1788204862.697508:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204862.697508:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 00000100:00000040:2.0:1788204862.697514:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697518:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204862.697520:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204862.697521:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204862.697523:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205905152, offset 0 00000400:00000200:2.0:1788204862.697525:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697528:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697532:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204862.697534:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204862.697536:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204862.697537:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204862.697539:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697541:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58927:58927:256:4294967295] 192.168.201.151@tcp LPNI seq info [58907:58907:8:4294967295] 00000400:00000200:2.0:1788204862.697543:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697547:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204862.697550:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697552:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204862.697555:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204862.697557:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204862.697560:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204862.697562:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204862.697564:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697565:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204862.697568:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204862.697571:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204862.697574:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204862.697576:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204862.697578:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204862.697581:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697589:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.697592:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697600:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697601:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204862.697604:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204862.697608:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204862.697612:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204862.697614:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300990107 00000100:00000001:2.0:1788204862.697617:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.697619:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300990107 00000100:00000001:2.0:1788204862.697622:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.697624:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204862.697626:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697628:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204862.697631:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204862/real 1788204862] req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204862.697639:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204862.697640:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204862.697642:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204862.697644:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204862.697646:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204862.697648:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204862.697651:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204862.697659:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697665:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204862.697667:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697669:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.697670:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204862.697674:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697681:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204862.697684:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697690:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204862.697692:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697693:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.697694:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697695:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204862.697697:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204862.697699:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204862.697701:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204862.697702:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204862.697704:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204862.697709:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905152:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204862.697712:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204862.697714:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697717:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204862.697719:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697720:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204862.697721:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204862.697723:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204862.697724:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905280:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204862.697726:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204862.697727:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204862.697728:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697729:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204862.697732:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204862.697733:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204862.697736:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204862.697738:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204862.697740:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204862.697742:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905280, portal 4 00000100:00000001:2.0:1788204862.697743:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204862.697745:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:2.0:1788204862.697748:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697752:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204862.697754:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204862.697755:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204862.697757:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205905280, offset 0 00000400:00000200:2.0:1788204862.697759:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697761:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697763:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204862.697766:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204862.697767:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204862.697769:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204862.697771:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697773:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58928:58928:256:4294967295] 192.168.201.151@tcp LPNI seq info [58908:58908:8:4294967295] 00000400:00000200:2.0:1788204862.697775:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697779:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204862.697782:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697784:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204862.697786:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204862.697788:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204862.697791:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204862.697792:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204862.697795:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697796:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204862.697798:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204862.697800:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204862.697802:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204862.697803:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204862.697805:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204862.697807:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697811:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.697814:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697818:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697819:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204862.697821:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204862.697823:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204862.697824:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204862.697826:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300989993 00000100:00000001:2.0:1788204862.697827:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.697829:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300989993 00000100:00000001:2.0:1788204862.697830:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.697831:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204862.697832:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.697833:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204862.697835:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204862/real 1788204862] req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204862.697839:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204862.697840:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204862.697842:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204862.697843:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204862.697844:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204862.697845:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204862.697847:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204862.697851:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697854:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204862.697855:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697856:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.697856:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204862.697859:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697863:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204862.697865:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697868:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204862.697870:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697870:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.697871:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.697872:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204862.697874:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204862.697875:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204862.697877:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204862.697878:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204862.697881:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204862.697888:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905280:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204862.697892:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204862.697895:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697913:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204862.697914:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697916:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204862.697918:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204862.697920:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204862.697923:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905408:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204862.697926:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204862.697929:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204862.697930:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204862.697932:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204862.697935:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204862.697937:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204862.697939:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204862.697941:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204862.697943:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204862.697945:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905408, portal 4 00000100:00000001:2.0:1788204862.697946:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204862.697947:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 00000100:00000040:2.0:1788204862.697952:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.697956:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204862.697957:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204862.697958:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204862.697960:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205905408, offset 0 00000400:00000200:2.0:1788204862.697962:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697964:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697966:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204862.697968:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204862.697970:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204862.697971:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204862.697973:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204862.697975:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58929:58929:256:4294967295] 192.168.201.151@tcp LPNI seq info [58909:58909:8:4294967295] 00000400:00000200:2.0:1788204862.697977:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204862.697981:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204862.697984:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697986:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204862.697988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204862.697990:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788204862.697993:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204862.697994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788204862.697997:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204862.697999:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204862.698001:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204862.698003:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204862.698006:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204862.698007:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204862.698010:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204862.698013:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698020:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.698022:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698029:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.698030:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204862.698034:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204862.698036:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204862.698039:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204862.698041:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300989999 00000100:00000001:2.0:1788204862.698043:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.698045:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300989999 00000100:00000001:2.0:1788204862.698047:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300994049 00000100:00000001:2.0:1788204862.698049:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204862.698051:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.698052:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204862.698055:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204862/real 1788204862] req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204862.698060:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204862.698061:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204862.698063:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204862.698066:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204862.698067:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204862.698070:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204862.698073:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204862.698081:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698087:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204862.698089:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698091:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.698092:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204862.698096:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698103:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204862.698106:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698110:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204862.698112:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698113:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204862.698114:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698115:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204862.698116:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204862.698117:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204862.698119:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204862.698121:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204862.698125:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204862.698133:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905408:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204862.698136:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698138:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.698141:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205905152/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698148:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204862.698150:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204862.698152:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204862.698154:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204862.698156:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204862.698157:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204862.698159:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204862.698161:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:2.0:1788204862.698163:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204862.698164:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204862.698165:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:2.0:1788204862.698167:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204862.698168:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698170:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.698172:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205905280/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698176:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204862.698177:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204862.698178:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204862.698180:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204862.698180:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204862.698181:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204862.698182:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204862.698183:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:2.0:1788204862.698184:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204862.698184:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204862.698185:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:2.0:1788204862.698187:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204862.698188:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698189:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204862.698192:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205905408/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204862.698197:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204862.698199:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204862.698200:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204862.698202:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204862.698203:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204862.698204:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204862.698206:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204862.698208:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:2.0:1788204862.698210:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204862.698211:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204862.698212:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000001:2.0:1788204862.698215:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204862.698216:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698218:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204862.698221:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204862.698223:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204862.698225:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204862.698226:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204862.888492:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204862.888494:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204862.888497:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888502:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204862.888503:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888505:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888506:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204862.888510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204862.888514:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204862.888515:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204862.888516:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888517:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204862.888518:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888519:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888520:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204862.888524:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204862.888525:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204862.888526:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888527:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204862.888528:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888528:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204862.888530:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204862.888544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204862.888546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204862.888548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204863.208482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204863.208489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204863.208491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204863.208496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204863.208500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204863.208504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204863.208506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204863.208509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204863.208511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204863.656502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204863.656512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204863.656519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204863.656527:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204863.656531:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204863.656536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204863.656539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204863.656542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204863.656545:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204863.720495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204863.720495:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204863.720504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204863.720507:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204863.720511:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204863.720513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204863.720515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204863.720518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204863.720520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204863.720522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204863.720524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204863.720527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204863.720527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204863.720530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204863.720534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204863.720536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204863.720539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204863.720541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204863.912521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204863.912529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204863.912534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204863.912535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204863.912537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204863.912538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204863.912541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204863.912542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204863.912588:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204863.912592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204863.912596:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204864.232503:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204864.232512:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204864.232514:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204864.232518:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204864.232521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204864.232525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204864.232528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204864.232531:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204864.232533:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204864.680484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204864.680493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204864.680496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204864.680502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204864.680507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204864.680511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204864.680514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204864.680517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204864.680521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204864.744479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204864.744489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204864.744492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204864.744497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1788204864.744500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204864.744503:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204864.744507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204864.744507:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204864.744512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204864.744513:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204864.744516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204864.744518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204864.744520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204864.744522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204864.744525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204864.744528:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204864.744531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204864.744534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204864.936508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204864.936518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204864.936524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204864.936525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204864.936529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204864.936530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204864.936533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204864.936534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204864.936556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204864.936559:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204864.936562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204865.256490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204865.256500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204865.256503:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204865.256509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204865.256513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204865.256516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204865.256518:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204865.256520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204865.256523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204865.704466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204865.704476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204865.704479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204865.704486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204865.704489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204865.704493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204865.704496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204865.704498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204865.704501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204865.768482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204865.768482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204865.768490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204865.768491:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204865.768493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204865.768495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204865.768501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204865.768502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204865.768505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204865.768507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204865.768510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204865.768511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204865.768514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204865.768517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204865.768520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204865.768523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204865.768527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204865.768530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204865.960505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204865.960516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204865.960523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204865.960524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204865.960528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204865.960529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204865.960533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204865.960534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204865.960591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204865.960596:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204865.960600:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204866.280476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204866.280486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204866.280489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204866.280495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204866.280500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204866.280504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204866.280507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204866.280510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204866.280513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204866.728443:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204866.728451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204866.728453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204866.728458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204866.728461:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204866.728465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204866.728470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204866.728473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204866.728476:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204866.792442:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204866.792442:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204866.792450:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204866.792450:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204866.792453:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204866.792454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204866.792461:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204866.792462:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204866.792466:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204866.792466:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204866.792471:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204866.792472:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204866.792474:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204866.792476:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204866.792479:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204866.792482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204866.792485:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204866.792487:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204866.984537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204866.984547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204866.984561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204866.984562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204866.984565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204866.984567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204866.984571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204866.984572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204866.984590:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204866.984593:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204866.984595:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204867.304497:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300994049, at 4300994560 00000100:00000040:1.0:1788204867.304497:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300994049, at 4300994560 00000100:00000001:3.0:1788204867.304499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204867.304506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204867.304510:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204867.304513:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00100000:3.0:1788204867.304515:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204867.304516:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000001:3.0:1788204867.304521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204867.304527:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.304531:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204867.304534:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.304538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.752521:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204867.752532:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204867.752534:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204867.752540:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204867.752544:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204867.752549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204867.752552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204867.752556:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204867.752559:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204867.816491:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204867.816492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204867.816493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204867.816500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204867.816500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204867.816502:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204867.816502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204867.816504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816506:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788204867.816507:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204867.816509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204867.816511:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204867.816511:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204867.816516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204867.816517:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204867.816517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204867.816521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.816522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788204867.816527:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6322 00000100:00000001:3.0:1788204867.816527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.816531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204867.816534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204867.816539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204867.816539:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204867.816544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204867.816548:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204867.816553:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204867.816556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816559:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.816563:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816565:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204867.816569:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204867.816572:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204867.816575:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204867.816580:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204867.816583:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204867.816585:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204867.816589:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816590:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.816592:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816593:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204867.816596:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204867.816598:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816600:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204867.816602:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204867.816605:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204867.816607:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816609:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204867.816611:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204867.816621:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788204867.816623:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204867.816627:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204867.816629:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816631:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204867.816634:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204867.816646:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204867.816656:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788204867.816663:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204867.816672:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204867.816679:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204867.816683:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204867.816687:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204867.816711:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204867.816714:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204867.816718:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204867.816719:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204867.816721:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204867.816724:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204867.816726:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6322 00000400:00000200:1.0:1788204867.816731:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204867.816735:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204867.816738:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204867.816741:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204867.816744:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204867.816745:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.816749:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204867.816750:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204867.816751:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204867.816754:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204867.816754:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788204867.816757:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204867.816759:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204867.816761:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204867.816763:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204867.816763:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204867.816767:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204867.816773:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816776:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.816780:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204867.816782:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204867.816783:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204867.816784:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204867.816786:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204867.816788:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788204867.816789:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:2.0:1788204867.816792:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204867.816794:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.816796:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204867.816797:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788204867.816802:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205905536:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788204867.816804:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204867.816807:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204867.816808:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.816825:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204867.816828:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816830:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204867.816831:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1788204867.816835:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000010:2.0:1788204867.816837:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 00000008:00000001:1.0:1788204867.816839:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.816840:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816842:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204867.816847:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000400:00000200:2.0:1788204867.816848:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788204867.816851:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204867.816854:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204867.816855:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.816856:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1788204867.816856:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 02000000:00000001:1.0:1788204867.816859:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204867.816862:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204867.816866:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000400:00000010:2.0:1788204867.816868:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000001:1.0:1788204867.816869:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204867.816877:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1788204867.816880:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905536, portal 25 00000100:00000001:1.0:1788204867.816885:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204867.816886:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204867.816888:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.816890:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:1.0:1788204867.816898:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204867.816900:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204867.816917:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204867.816921:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204867.816921:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788204867.816927:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204867.816929:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1788204867.816929:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788204867.816932:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204867.816935:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:2.0:1788204867.816935:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00080000:1.0:1788204867.816940:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6322 00000400:00000200:1.0:1788204867.816943:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:2.0:1788204867.816943:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205905536, offset 0 00000400:00000200:1.0:1788204867.816948:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204867.816950:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204867.816952:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816956:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204867.816957:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788204867.816960:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816963:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204867.816964:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204867.816965:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204867.816966:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204867.816968:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204867.816971:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204867.816972:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204867.816974:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816975:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.816977:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816978:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204867.816978:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788204867.816983:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204867.816985:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204867.816986:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.816988:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000001:1.0:1788204867.816989:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204867.816993:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204867.816996:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.816997:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204867.817001:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788204867.817003:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788204867.817005:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000008:00100000:1.0:1788204867.817008:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:2.0:1788204867.817009:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817013:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58930:58930:256:4294967295] 192.168.201.151@tcp LPNI seq info [58910:58910:8:4294967295] 00000008:00000001:1.0:1788204867.817019:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.817020:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204867.817024:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204867.817029:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204867.817033:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:2.0:1788204867.817036:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 02000000:00000001:1.0:1788204867.817039:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204867.817042:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204867.817043:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204867.817047:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204867.817049:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204867.817051:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204867.817051:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 02000000:00000010:1.0:1788204867.817058:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000800:00000200:2.0:1788204867.817062:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204867.817063:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204867.817069:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00100000:1.0:1788204867.817070:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1788204867.817079:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204867.817083:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1788204867.817085:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1788204867.817087:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204867.817095:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1788204867.817095:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817100:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204867.817107:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788204867.817110:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.817112:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1788204867.817115:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204867.817118:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1788204867.817118:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000001:1.0:1788204867.817122:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1788204867.817122:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00080000:1.0:1788204867.817126:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204867.817128:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204867.817129:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1788204867.817132:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6322 00000100:00000200:2.0:1788204867.817136:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204867.817138:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204867.817144:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204867.817148:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204867.817152:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204867.817153:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.817158:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204867.817159:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.817165:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204867.817166:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.817167:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204867.817169:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1788204867.817172:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204867.817174:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204867.817174:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204867.817176:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204867.817177:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204867.817179:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:2.0:1788204867.817180:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204867.817181:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.817185:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204867.817187:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204867.817191:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788204867.817191:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788204867.817194:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204867.817197:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204867.817197:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4300999169 00000100:00000001:1.0:1788204867.817201:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204867.817203:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204867.817205:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204867.817206:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.817208:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000010:2.0:1788204867.817208:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1660. 00000008:00100000:1.0:1788204867.817211:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204867.817214:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.817215:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204867.817218:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000100:00000001:2.0:1788204867.817218:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1788204867.817219:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204867.817224:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204867.817225:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.817226:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204867.817227:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204867.817228:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4300999169 02000000:00000001:1.0:1788204867.817229:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204867.817233:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000010:2.0:1788204867.817235:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1600. 00000100:00000001:1.0:1788204867.817236:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817242:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00100000:1.0:1788204867.817243:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204867.817249:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204867.817251:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204867.817254:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204867.817255:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817261:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788204867.817262:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204867.817268:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204867.817270:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204867/real 1788204867] req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204867.817273:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6332) 00000400:00000001:2.0:1788204867.817283:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204867.817285:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204867.817288:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204867.817299:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204867.817301:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204867.817303:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204867.817307:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204867.817315:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817322:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204867.817324:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817326:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817328:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204867.817331:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817336:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204867.817339:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817344:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204867.817346:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817346:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817347:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817349:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204867.817350:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204867.817352:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204867.817355:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204867.817356:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204867.817359:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204867.817390:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205905536:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204867.817406:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817408:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.817411:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205905536/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817418:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204867.817420:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204867.817422:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204867.817425:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204867.817427:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204867.817428:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204867.817430:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204867.817432:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:2.0:1788204867.817435:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204867.817436:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204867.817437:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:2.0:1788204867.817440:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204867.817441:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817444:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817454:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204867.817455:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817457:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204867.817459:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204867.817460:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204867.817464:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817471:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204867.817473:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817475:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204867.817477:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204867.817479:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204867.817482:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905664:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204867.817485:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204867.817487:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204867.817488:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817490:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204867.817492:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204867.817494:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.817497:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204867.817498:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204867.817501:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204867.817503:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905664, portal 10 00000100:00000001:2.0:1788204867.817504:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204867.817505:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:2.0:1788204867.817511:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817515:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204867.817517:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204867.817518:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204867.817519:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205905664, offset 0 00000400:00000200:2.0:1788204867.817522:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817524:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817527:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204867.817529:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204867.817531:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204867.817532:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204867.817534:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817536:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58931:58931:256:4294967295] 192.168.201.151@tcp LPNI seq info [58911:58911:8:4294967295] 00000400:00000200:2.0:1788204867.817538:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817543:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204867.817546:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817548:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204867.817551:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204867.817553:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204867.817556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204867.817558:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204867.817560:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817562:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204867.817564:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204867.817566:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204867.817568:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204867.817569:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204867.817571:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204867.817573:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817578:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.817580:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817583:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817584:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204867.817586:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204867.817589:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204867.817591:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204867.817592:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300995227 00000100:00000001:2.0:1788204867.817594:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817595:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300995227 00000100:00000001:2.0:1788204867.817596:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817598:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204867.817599:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817600:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204867.817603:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204867/real 1788204867] req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204867.817607:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204867.817607:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204867.817608:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204867.817609:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204867.817610:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204867.817611:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204867.817613:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204867.817618:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817622:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204867.817623:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817624:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817625:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204867.817626:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817630:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204867.817632:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817635:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204867.817637:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817638:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817638:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817639:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204867.817641:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204867.817642:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204867.817644:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204867.817645:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204867.817648:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204867.817652:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905664:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204867.817654:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204867.817656:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817664:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204867.817665:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817666:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204867.817667:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204867.817668:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204867.817670:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905792:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204867.817672:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204867.817672:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204867.817673:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817674:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204867.817676:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204867.817677:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.817679:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204867.817680:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204867.817682:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204867.817684:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905792, portal 4 00000100:00000001:2.0:1788204867.817685:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204867.817686:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:2.0:1788204867.817690:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817694:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204867.817695:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204867.817696:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204867.817698:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205905792, offset 0 00000400:00000200:2.0:1788204867.817700:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817701:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817703:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204867.817704:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204867.817706:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204867.817707:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204867.817709:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817711:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58932:58932:256:4294967295] 192.168.201.151@tcp LPNI seq info [58912:58912:8:4294967295] 00000400:00000200:2.0:1788204867.817714:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817717:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204867.817720:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817722:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204867.817724:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204867.817725:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204867.817728:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204867.817729:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204867.817732:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817733:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204867.817735:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204867.817737:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204867.817739:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204867.817740:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204867.817741:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204867.817743:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817747:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.817749:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817753:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817754:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204867.817755:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204867.817757:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204867.817758:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204867.817759:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300995113 00000100:00000001:2.0:1788204867.817761:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817762:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300995113 00000100:00000001:2.0:1788204867.817764:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817765:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204867.817767:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817768:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204867.817770:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204867/real 1788204867] req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204867.817775:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204867.817775:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204867.817776:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204867.817777:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204867.817778:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204867.817779:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204867.817781:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204867.817785:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817789:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204867.817790:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817791:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817792:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204867.817793:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817797:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204867.817799:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817802:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204867.817803:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817804:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817805:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817806:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204867.817807:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204867.817809:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204867.817825:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204867.817827:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204867.817830:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204867.817835:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905792:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204867.817836:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204867.817838:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817842:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204867.817843:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817845:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204867.817846:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204867.817847:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204867.817848:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905920:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204867.817850:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204867.817851:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204867.817852:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204867.817853:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204867.817857:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204867.817858:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204867.817860:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204867.817862:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204867.817864:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204867.817866:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205905920, portal 4 00000100:00000001:2.0:1788204867.817867:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204867.817868:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000040:2.0:1788204867.817872:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817876:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204867.817878:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204867.817879:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204867.817880:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205905920, offset 0 00000400:00000200:2.0:1788204867.817883:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817885:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817887:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204867.817889:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204867.817891:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204867.817892:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204867.817894:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204867.817896:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58933:58933:256:4294967295] 192.168.201.151@tcp LPNI seq info [58913:58913:8:4294967295] 00000400:00000200:2.0:1788204867.817898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204867.817902:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204867.817905:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817906:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204867.817909:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204867.817911:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788204867.817913:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204867.817915:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788204867.817917:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204867.817918:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204867.817920:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204867.817922:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204867.817924:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204867.817925:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204867.817927:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204867.817929:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817933:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.817935:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817939:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817940:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204867.817942:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204867.817944:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204867.817946:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204867.817947:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4300995119 00000100:00000001:2.0:1788204867.817949:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817950:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4300995119 00000100:00000001:2.0:1788204867.817952:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4300999169 00000100:00000001:2.0:1788204867.817953:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204867.817954:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.817955:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204867.817957:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204867/real 1788204867] req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204867.817961:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204867.817962:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204867.817963:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204867.817964:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204867.817965:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204867.817966:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204867.817968:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204867.817972:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817976:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204867.817977:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817978:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817979:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204867.817981:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817984:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204867.817986:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.817991:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204867.817992:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817993:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204867.817994:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.817995:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204867.817997:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204867.817998:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204867.817999:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204867.818001:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204867.818003:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204867.818006:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205905920:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204867.818008:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.818009:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.818011:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205905664/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.818014:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204867.818015:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204867.818017:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204867.818018:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204867.818019:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204867.818020:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204867.818021:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204867.818023:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:2.0:1788204867.818024:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204867.818025:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204867.818026:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:2.0:1788204867.818028:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204867.818029:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.818030:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.818032:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205905792/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.818036:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204867.818036:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204867.818037:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204867.818039:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204867.818039:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204867.818040:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204867.818041:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204867.818042:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:2.0:1788204867.818043:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204867.818043:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204867.818044:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:2.0:1788204867.818045:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204867.818046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.818047:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204867.818048:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205905920/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204867.818051:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204867.818052:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204867.818053:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204867.818054:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204867.818055:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204867.818055:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204867.818056:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204867.818057:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:2.0:1788204867.818058:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204867.818059:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204867.818059:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:2.0:1788204867.818060:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204867.818061:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.818062:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204867.818064:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204867.818064:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204867.818065:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204867.818067:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204868.008505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204868.008515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204868.008523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204868.008525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204868.008529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204868.008530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204868.008534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204868.008535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204868.008591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204868.008595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204868.008599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204868.328464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204868.328472:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204868.328474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204868.328479:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204868.328482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204868.328487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.328489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204868.328491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.328494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204868.776503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204868.776513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204868.776516:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204868.776523:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204868.776527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204868.776532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204868.776535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204868.776538:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204868.776542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204868.840432:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204868.840440:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204868.840443:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204868.840450:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204868.840454:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204868.840460:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204868.840464:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204868.840466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204868.840467:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.840475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204868.840476:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204868.840483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204868.840490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204868.840494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204868.840498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.840501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204868.840504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.840507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204869.032517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204869.032524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204869.032532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204869.032533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204869.032535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204869.032536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204869.032539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204869.032540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204869.032556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204869.032558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204869.032560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204869.352470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204869.352475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204869.352476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.352480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204869.352483:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204869.352486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.352488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204869.352489:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.352492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204869.800481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204869.800489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204869.800491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204869.800496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204869.800499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204869.800503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204869.800505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204869.800507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204869.800510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204869.864479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204869.864480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204869.864487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204869.864488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204869.864490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.864491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204869.864497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204869.864497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204869.864504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204869.864504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204869.864509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204869.864510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.864513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204869.864516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.864520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204869.864521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204869.864524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204869.864527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204870.056513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204870.056523:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204870.056526:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204870.056530:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056533:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204870.056536:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056538:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056541:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204870.056546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204870.056550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204870.056551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204870.056604:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204870.056612:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204870.056617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204870.376424:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204870.376432:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204870.376435:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204870.376440:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204870.376444:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204870.376447:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204870.376451:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204870.376453:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204870.376456:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204870.824459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204870.824470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204870.824472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204870.824480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204870.824484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204870.824489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204870.824493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204870.824496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204870.824500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204870.888450:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204870.888451:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204870.888457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204870.888458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204870.888459:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204870.888461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204870.888467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204870.888467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204870.888473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204870.888473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204870.888479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204870.888480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204870.888486:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204870.888490:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204870.888493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204870.888494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204870.888496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204870.888499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204871.080525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204871.080532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204871.080536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204871.080537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204871.080540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204871.080541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204871.080543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204871.080544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204871.080562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204871.080564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204871.080566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204871.400484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204871.400491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204871.400493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204871.400498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204871.400502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204871.400506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204871.400508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204871.400511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204871.400514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204871.848503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204871.848512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204871.848514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204871.848521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204871.848526:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204871.848532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204871.848535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204871.848537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204871.848540:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204871.912449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204871.912450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204871.912456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204871.912456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204871.912459:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204871.912460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204871.912465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204871.912466:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204871.912470:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204871.912470:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204871.912474:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204871.912475:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204871.912476:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204871.912479:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204871.912481:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204871.912482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204871.912485:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204871.912488:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204872.104495:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204872.104497:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204872.104500:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104504:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204872.104506:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104508:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104510:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204872.104514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204872.104517:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204872.104518:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204872.104519:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104520:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204872.104521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204872.104526:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204872.104527:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204872.104527:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104528:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204872.104529:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104530:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204872.104531:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204872.104594:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204872.104601:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204872.104606:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204872.424479:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4300999169, at 4300999680 00000100:00000040:1.0:1788204872.424479:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4300999169, at 4300999680 00000100:00000001:3.0:1788204872.424480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204872.424486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204872.424489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204872.424495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204872.424497:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000010:0.0:1788204872.424499:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:3.0:1788204872.424500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204872.424505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204872.424509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204872.424511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204872.424514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204872.872475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204872.872481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204872.872483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204872.872486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204872.872489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204872.872492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204872.872494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204872.872495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204872.872497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.936515:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204872.936515:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204872.936517:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204872.936525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204872.936526:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204872.936530:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204872.936530:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204872.936531:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936536:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788204872.936540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204872.936545:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204872.936545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204872.936545:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204872.936550:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204872.936552:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204872.936555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204872.936557:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204872.936558:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.936559:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204872.936565:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204872.936566:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204872.936567:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6327 00000100:00000001:3.0:1788204872.936569:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204872.936577:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204872.936584:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204872.936593:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204872.936596:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204872.936599:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936602:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204872.936604:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936606:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204872.936609:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204872.936611:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204872.936613:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204872.936616:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204872.936617:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204872.936618:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204872.936620:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936622:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204872.936623:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936624:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204872.936626:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204872.936628:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936629:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204872.936631:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204872.936638:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204872.936640:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936642:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204872.936643:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204872.936651:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204872.936653:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204872.936656:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204872.936657:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936659:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204872.936661:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204872.936664:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204872.936671:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204872.936676:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204872.936682:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204872.936689:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204872.936692:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204872.936695:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204872.936713:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204872.936716:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204872.936719:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204872.936720:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204872.936731:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204872.936734:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204872.936737:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6327 00000400:00000200:0.0:1788204872.936742:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204872.936745:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204872.936748:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204872.936750:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788204872.936752:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204872.936756:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204872.936757:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204872.936759:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204872.936766:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936772:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:2.0:1788204872.936772:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204872.936782:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204872.936786:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204872.936792:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204872.936796:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:2.0:1788204872.936801:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204872.936802:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204872.936805:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204872.936808:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788204872.936811:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1788204872.936816:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:2.0:1788204872.936817:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204872.936823:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204872.936827:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205906048:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204872.936828:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204872.936834:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788204872.936838:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936840:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204872.936842:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1788204872.936843:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204872.936846:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204872.936849:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204872.936854:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.936854:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204872.936859:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204872.936864:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1788204872.936865:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788204872.936866:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936868:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204872.936873:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788204872.936875:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788204872.936879:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936883:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.936888:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:0.0:1788204872.936889:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000010:2.0:1788204872.936894:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000020:00000040:0.0:1788204872.936895:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204872.936901:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1788204872.936903:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 02000000:00000001:0.0:1788204872.936904:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.936911:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204872.936914:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204872.936916:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204872.936917:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906048, portal 25 00000100:00000001:2.0:1788204872.936920:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:0.0:1788204872.936921:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:2.0:1788204872.936923:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204872.936926:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204872.936932:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204872.936942:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788204872.936942:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204872.936949:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204872.936958:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.936970:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204872.936973:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204872.936977:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1788204872.936978:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204872.936980:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:2.0:1788204872.936984:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000001:0.0:1788204872.936989:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204872.936992:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1788204872.936992:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205906048, offset 0 00000100:00000001:0.0:1788204872.937001:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204872.937004:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6327 00000400:00000200:0.0:1788204872.937016:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.937016:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204872.937026:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204872.937029:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204872.937030:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.937034:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788204872.937036:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204872.937039:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.937043:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204872.937045:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1788204872.937048:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:0.0:1788204872.937049:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204872.937056:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204872.937058:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1788204872.937061:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:2.0:1788204872.937065:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788204872.937071:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204872.937071:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1788204872.937079:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:2.0:1788204872.937082:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:0.0:1788204872.937089:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788204872.937094:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58934:58934:256:4294967295] 192.168.201.151@tcp LPNI seq info [58914:58914:8:4294967295] 00000100:00000001:0.0:1788204872.937095:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204872.937100:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.937109:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204872.937112:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:2.0:1788204872.937114:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204872.937120:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204872.937123:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204872.937124:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204872.937129:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204872.937135:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.937135:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204872.937142:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204872.937151:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.0:1788204872.937157:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000008:00000001:0.0:1788204872.937159:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.937163:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204872.937165:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00000010:0.0:1788204872.937169:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204872.937174:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.0:1788204872.937181:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204872.937183:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204872.937188:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788204872.937190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000100:00000001:0.0:1788204872.937192:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204872.937195:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204872.937201:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204872.937204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204872.937208:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204872.937228:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.937232:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 02000000:00000010:0.0:1788204872.937233:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204872.937239:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.937240:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204872.937252:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1788204872.937258:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204872.937261:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000001:0.0:1788204872.937264:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204872.937266:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204872.937269:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204872.937274:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:0.0:1788204872.937277:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204872.937281:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204872.937283:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1788204872.937284:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204872.937287:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204872.937288:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204872.937291:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:2.0:1788204872.937291:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204872.937295:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6327 00000400:00000200:0.0:1788204872.937305:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204872.937310:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204872.937322:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204872.937331:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204872.937333:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204872.937336:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204872.937342:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204872.937343:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204872.937348:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204872.937354:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204872.937358:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:2.0:1788204872.937385:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937388:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1788204872.937391:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204872.937393:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:2.0:1788204872.937395:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204872.937396:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204872.937405:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204872.937411:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204872.937417:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204872.937418:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301004290 00000100:00000001:0.0:1788204872.937428:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204872.937432:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:2.0:1788204872.937432:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1900. 00000100:00000001:0.0:1788204872.937440:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204872.937442:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204872.937446:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204872.937447:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204872.937448:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204872.937453:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204872.937456:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204872.937458:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204872.937460:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204872.937460:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301004290 00000008:00100000:0.0:1788204872.937464:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000010:2.0:1788204872.937467:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a80. 00000008:00000001:0.0:1788204872.937470:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937473:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788204872.937475:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937479:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1788204872.937482:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204872.937485:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204872.937487:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:2.0:1788204872.937490:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788204872.937493:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204872.937496:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204872.937500:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204872/real 1788204872] req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204872.937501:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204872.937504:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204872.937507:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204872.937513:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204872.937519:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788204872.937530:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1788204872.937531:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788204872.937536:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204872.937542:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000100:00000001:0.0:1788204872.937544:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204872.937548:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000100:00000001:0.0:1788204872.937550:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204872.937552:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204872.937558:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1788204872.937559:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204872.937566:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204872.937603:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204872.937609:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204872.937610:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6337) 00000100:00000001:2.0:1788204872.937622:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204872.937626:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937629:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.937630:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204872.937634:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937639:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204872.937642:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937647:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204872.937648:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937649:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.937650:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937652:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204872.937653:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204872.937656:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204872.937659:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204872.937660:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204872.937663:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204872.937667:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205906048:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204872.937671:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937673:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.937675:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205906048/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937679:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204872.937682:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204872.937684:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204872.937686:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204872.937688:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204872.937690:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204872.937691:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204872.937693:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:2.0:1788204872.937696:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204872.937698:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204872.937700:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788204872.937702:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204872.937703:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937705:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937712:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204872.937713:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937716:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204872.937717:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204872.937718:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204872.937720:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937737:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204872.937738:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.937740:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204872.937742:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204872.937743:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204872.937745:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906176:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204872.937748:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204872.937749:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204872.937751:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.937752:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204872.937755:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204872.937756:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.937759:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204872.937761:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204872.937763:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204872.937765:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906176, portal 10 00000100:00000001:2.0:1788204872.937767:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204872.937768:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788204872.937773:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937777:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204872.937779:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204872.937780:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204872.937781:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205906176, offset 0 00000400:00000200:2.0:1788204872.937784:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204872.937787:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.937789:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204872.937791:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204872.937793:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204872.937795:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204872.937796:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204872.937798:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58935:58935:256:4294967295] 192.168.201.151@tcp LPNI seq info [58915:58915:8:4294967295] 00000400:00000200:2.0:1788204872.937801:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.937805:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204872.937807:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.937809:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204872.937812:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204872.937814:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204872.937818:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204872.937820:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204872.937822:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.937823:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204872.937826:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204872.937828:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204872.937830:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204872.937831:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204872.937833:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204872.937835:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937840:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.937842:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937848:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937849:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204872.937852:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204872.937856:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204872.937859:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204872.937861:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301000347 00000100:00000001:2.0:1788204872.937864:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.937866:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301000347 00000100:00000001:2.0:1788204872.937869:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.937871:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204872.937873:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.937874:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204872.937879:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204872/real 1788204872] req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204872.937887:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204872.937888:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204872.937890:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204872.937892:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204872.937894:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204872.937896:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204872.937899:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204872.937904:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937908:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204872.937909:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937910:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.937911:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204872.937913:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937917:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204872.937920:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937923:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204872.937925:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937925:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.937926:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.937927:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204872.937929:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204872.937930:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204872.937932:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204872.937933:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204872.937935:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204872.937939:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906176:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204872.937942:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204872.937944:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937953:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204872.937954:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.937956:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204872.937957:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204872.937958:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204872.937959:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906304:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204872.937961:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204872.937963:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204872.937964:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.937965:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204872.937967:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204872.937968:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.937971:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204872.937972:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204872.937975:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204872.937977:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906304, portal 4 00000100:00000001:2.0:1788204872.937978:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204872.937979:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788204872.937983:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.937987:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204872.937988:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204872.937990:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204872.937991:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205906304, offset 0 00000400:00000200:2.0:1788204872.937993:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204872.937996:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.937997:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204872.937999:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204872.938001:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204872.938003:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204872.938004:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204872.938007:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58936:58936:256:4294967295] 192.168.201.151@tcp LPNI seq info [58916:58916:8:4294967295] 00000400:00000200:2.0:1788204872.938009:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.938013:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204872.938016:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.938018:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204872.938021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204872.938023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204872.938025:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204872.938027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204872.938030:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.938031:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204872.938033:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204872.938035:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204872.938037:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204872.938038:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204872.938039:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204872.938041:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938048:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.938052:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938058:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.938060:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204872.938063:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204872.938066:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204872.938069:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204872.938071:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301000233 00000100:00000001:2.0:1788204872.938073:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.938075:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301000233 00000100:00000001:2.0:1788204872.938078:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.938080:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204872.938082:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.938084:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204872.938087:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204872/real 1788204872] req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204872.938095:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204872.938097:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204872.938098:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204872.938101:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204872.938102:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204872.938104:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204872.938107:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204872.938115:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938122:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204872.938124:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938126:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.938127:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204872.938130:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938135:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204872.938138:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938142:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204872.938143:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938144:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.938145:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938146:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204872.938147:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204872.938149:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204872.938151:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204872.938153:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204872.938155:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204872.938161:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906304:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204872.938164:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204872.938167:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938173:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204872.938175:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.938178:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204872.938179:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204872.938181:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204872.938184:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906432:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204872.938187:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204872.938189:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204872.938191:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204872.938193:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204872.938196:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204872.938198:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204872.938201:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204872.938204:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204872.938207:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204872.938210:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906432, portal 4 00000100:00000001:2.0:1788204872.938211:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204872.938212:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788204872.938217:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938221:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204872.938222:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204872.938224:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204872.938226:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205906432, offset 0 00000400:00000200:2.0:1788204872.938228:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204872.938231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.938234:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204872.938235:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204872.938237:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204872.938239:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204872.938240:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204872.938242:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58937:58937:256:4294967295] 192.168.201.151@tcp LPNI seq info [58917:58917:8:4294967295] 00000400:00000200:2.0:1788204872.938245:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204872.938249:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204872.938251:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.938253:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204872.938255:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204872.938257:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788204872.938259:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204872.938261:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788204872.938263:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204872.938265:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204872.938267:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204872.938269:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204872.938271:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204872.938272:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204872.938274:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204872.938276:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938281:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.938285:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938291:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.938293:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204872.938296:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204872.938300:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204872.938303:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204872.938305:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301000239 00000100:00000001:2.0:1788204872.938307:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.938310:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301000239 00000100:00000001:2.0:1788204872.938312:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301004290 00000100:00000001:2.0:1788204872.938313:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204872.938316:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.938318:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204872.938322:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204872/real 1788204872] req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204872.938330:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204872.938331:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204872.938333:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204872.938336:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204872.938337:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204872.938339:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204872.938343:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204872.938349:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938353:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204872.938354:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938355:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.938356:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204872.938358:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938362:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204872.938381:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938386:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204872.938387:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938388:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204872.938389:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938390:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204872.938392:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204872.938393:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204872.938395:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204872.938396:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204872.938398:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204872.938403:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906432:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204872.938405:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938407:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.938408:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205906176/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938412:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204872.938412:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204872.938414:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204872.938415:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204872.938416:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204872.938417:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204872.938419:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204872.938420:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:2.0:1788204872.938422:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204872.938423:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204872.938424:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788204872.938425:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204872.938426:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938428:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.938430:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205906304/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938434:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204872.938435:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204872.938435:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204872.938436:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204872.938437:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204872.938438:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204872.938439:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204872.938439:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:2.0:1788204872.938441:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204872.938442:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204872.938442:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788204872.938444:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204872.938444:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938445:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204872.938447:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205906432/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204872.938450:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204872.938450:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204872.938451:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204872.938452:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204872.938453:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204872.938454:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204872.938454:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204872.938455:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:2.0:1788204872.938456:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204872.938457:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204872.938457:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788204872.938458:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204872.938459:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938460:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204872.938463:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204872.938464:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204872.938466:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204872.938468:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204873.128483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204873.128487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204873.128491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204873.128492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204873.128494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204873.128495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204873.128497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204873.128498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204873.128510:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204873.128511:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204873.128513:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204873.448457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204873.448463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204873.448466:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204873.448471:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204873.448475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204873.448479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204873.448482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204873.448485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204873.448488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204873.896447:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204873.896453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204873.896454:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204873.896458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204873.896461:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204873.896465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204873.896467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204873.896470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204873.896473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204873.960466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204873.960471:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204873.960473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204873.960477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204873.960480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204873.960480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204873.960485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204873.960486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204873.960488:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204873.960489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204873.960490:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204873.960495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204873.960496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204873.960501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204873.960503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204873.960505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204873.960507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204873.960509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204874.152468:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204874.152475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204874.152482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204874.152483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204874.152486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204874.152487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204874.152491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204874.152492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204874.152543:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204874.152546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204874.152549:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204874.472468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204874.472480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204874.472482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204874.472487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204874.472490:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204874.472495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204874.472498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204874.472501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204874.472503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204874.920483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204874.920490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204874.920493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204874.920499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204874.920502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204874.920506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204874.920509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204874.920512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204874.920516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204874.984458:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204874.984464:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204874.984467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204874.984473:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204874.984477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204874.984480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204874.984483:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204874.984486:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204874.984487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204874.984490:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204874.984492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204874.984495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204874.984498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204874.984501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204874.984503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204874.984505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204874.984507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204874.984510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204875.176495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204875.176499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204875.176503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204875.176503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204875.176506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204875.176506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204875.176508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204875.176509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204875.176522:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204875.176525:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204875.176528:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204875.496462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204875.496467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204875.496469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204875.496473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204875.496475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204875.496478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204875.496480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204875.496482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204875.496484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204875.944478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204875.944486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204875.944488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204875.944492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204875.944495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204875.944499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204875.944501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204875.944504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204875.944506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204876.008468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204876.008476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204876.008478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204876.008484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204876.008488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204876.008491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.008494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204876.008498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204876.008501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204876.008502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204876.008505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204876.008508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204876.008508:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204876.008511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.008513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204876.008514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204876.008516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204876.008519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204876.200459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204876.200464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204876.200468:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204876.200469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204876.200471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204876.200472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204876.200474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204876.200475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204876.200532:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204876.200536:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204876.200539:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204876.520472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204876.520479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204876.520480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204876.520485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204876.520487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204876.520490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.520492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204876.520494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.520497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204876.968463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204876.968468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204876.968470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204876.968474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204876.968476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204876.968479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204876.968481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204876.968482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204876.968484:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204877.032454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204877.032455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204877.032460:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204877.032461:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204877.032463:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204877.032465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204877.032468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204877.032470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204877.032474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204877.032477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204877.032480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.032482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204877.032485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204877.032489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204877.032492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204877.032493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204877.032496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.032499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204877.224495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204877.224500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204877.224504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204877.224505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204877.224508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204877.224509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204877.224512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204877.224514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204877.224534:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204877.224537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204877.224540:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204877.544446:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301004290, at 4301004800 00000100:00000001:3.0:1788204877.544447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204877.544450:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301004290, at 4301004800 00000100:00000001:3.0:1788204877.544452:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788204877.544460:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000010:0.0:1788204877.544462:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00000001:3.0:1788204877.544463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204877.544470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204877.544472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204877.544475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.544478:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204877.544480:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.544483:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204877.992432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204877.992439:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204877.992440:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204877.992444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204877.992447:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204877.992449:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204877.992451:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204877.992453:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204877.992455:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204878.056470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204878.056470:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204878.056478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204878.056478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204878.056480:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204878.056480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204878.056482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.056486:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204878.056489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204878.056490:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204878.056490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204878.056494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204878.056495:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204878.056496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204878.056497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204878.056501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.056501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204878.056502:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204878.056508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204878.056509:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204878.056509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204878.056513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204878.056517:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6332 00000100:00000001:0.0:1788204878.056519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204878.056526:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204878.056531:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204878.056533:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204878.056536:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056540:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.056543:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056546:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204878.056549:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204878.056551:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204878.056554:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204878.056557:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204878.056559:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204878.056562:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204878.056565:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056566:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.056568:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056570:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204878.056573:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204878.056576:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056579:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204878.056581:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204878.056584:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204878.056587:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.056589:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204878.056591:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204878.056598:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204878.056601:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204878.056605:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204878.056607:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.056609:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204878.056611:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204878.056615:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204878.056652:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204878.056657:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204878.056664:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204878.056671:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204878.056674:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204878.056677:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.056696:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204878.056698:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204878.056700:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204878.056701:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204878.056702:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204878.056704:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204878.056705:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6332 00000400:00000200:2.0:1788204878.056708:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204878.056711:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204878.056712:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204878.056715:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056717:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.056719:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056720:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204878.056722:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204878.056722:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1788204878.056726:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204878.056727:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788204878.056728:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204878.056730:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1788204878.056731:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204878.056734:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204878.056736:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204878.056738:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204878.056741:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056744:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.056748:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056752:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204878.056754:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788204878.056755:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204878.056756:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204878.056759:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056762:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.056762:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204878.056766:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204878.056767:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204878.056770:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204878.056772:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204878.056773:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.056777:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1788204878.056780:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205906560:192.168.201.151@tcp:250:kworker.0 00000008:00100000:2.0:1788204878.056783:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204878.056787:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788204878.056788:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.056792:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.056793:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:2.0:1788204878.056796:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000001:3.0:1788204878.056797:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204878.056802:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204878.056803:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204878.056807:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204878.056809:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204878.056810:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 00000100:00000001:2.0:1788204878.056811:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204878.056813:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204878.056815:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204878.056819:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204878.056820:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204878.056825:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 02000000:00000010:2.0:1788204878.056826:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000400:00000010:3.0:1788204878.056830:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:2.0:1788204878.056832:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204878.056837:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906560, portal 25 00000100:00100000:2.0:1788204878.056841:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204878.056841:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204878.056843:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788204878.056852:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204878.056856:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.056862:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204878.056863:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.056893:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788204878.056895:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.056898:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204878.056901:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204878.056902:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204878.056905:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205906560, offset 0 00000100:00080000:2.0:1788204878.056908:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204878.056911:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204878.056912:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204878.056916:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204878.056920:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204878.056923:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788204878.056926:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6332 00000400:00000200:3.0:1788204878.056931:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204878.056936:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204878.056939:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204878.056941:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204878.056943:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204878.056945:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58938:58938:256:4294967295] 192.168.201.151@tcp LPNI seq info [58918:58918:8:4294967295] 00000400:00000200:3.0:1788204878.056948:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204878.056953:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.056958:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:2.0:1788204878.056963:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204878.056965:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1788204878.056971:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204878.056972:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056977:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1788204878.056977:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:2.0:1788204878.056983:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.056988:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788204878.056988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204878.056994:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:3.0:1788204878.056994:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000020:00000040:2.0:1788204878.056997:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000800:00000200:3.1:1788204878.057001:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204878.057002:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:3.1:1788204878.057006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000020:00000040:2.0:1788204878.057007:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000100:3.0:1788204878.057011:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788204878.057012:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:3.0:1788204878.057014:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000020:00000001:2.0:1788204878.057015:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204878.057019:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788204878.057020:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.057023:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204878.057026:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788204878.057029:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.057031:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204878.057032:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000020:00000040:2.0:1788204878.057033:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000010:3.0:1788204878.057035:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00080000:2.0:1788204878.057037:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204878.057040:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057040:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788204878.057043:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788204878.057046:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204878.057047:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204878.057052:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.057055:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204878.057059:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:3.0:1788204878.057061:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788204878.057063:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.057065:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057066:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204878.057073:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204878.057078:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:3.0:1788204878.057082:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204878.057084:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204878.057085:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1788204878.057088:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057091:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.057092:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204878.057095:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204878.057096:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788204878.057100:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057100:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301009409 00000100:00000010:3.0:1788204878.057106:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4ae0. 02000000:00000010:2.0:1788204878.057109:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:3.0:1788204878.057112:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204878.057114:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057119:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301009409 00000100:00100000:2.0:1788204878.057123:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788204878.057124:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4780. 00000100:00000001:3.0:1788204878.057129:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204878.057133:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204878.057134:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204878.057136:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204878.057137:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057139:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788204878.057142:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.057146:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057150:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204878.057154:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204878.057161:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204878.057163:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204878.057165:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204878.057169:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:3.0:1788204878.057170:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788204878.057171:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6332 00000400:00000001:3.0:1788204878.057173:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1788204878.057176:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204878.057178:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:2.0:1788204878.057180:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204878.057183:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204878.057184:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000040:2.0:1788204878.057186:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204878.057188:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:2.0:1788204878.057189:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.057193:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.057195:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204878.057195:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1788204878.057199:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204878.057200:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204878.057202:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204878.057204:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000200:3.0:1788204878.057204:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204878.057208:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204878.057212:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204878.057218:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204878.057223:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204878.057228:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204878.057228:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057234:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204878.057238:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1788204878.057240:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204878.057241:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057246:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788204878.057248:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204878.057249:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204878.057254:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057255:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204878.057261:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204878.057266:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204878.057266:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204878.057271:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.057272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057274:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204878.057277:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788204878.057280:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788204878.057281:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057285:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204878.057286:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057289:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057290:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1788204878.057292:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204878.057292:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204878.057295:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1788204878.057296:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00080000:3.0:1788204878.057298:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788204878.057300:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204878.057302:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204878.057302:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204878.057306:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204878.057307:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788204878.057309:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1788204878.057312:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204878.057313:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204878.057318:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788204878.057323:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.057324:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205906560:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204878.057328:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204878.057330:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204878.057331:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.057336:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205906560/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204878.057343:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204878.057347:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057350:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1788204878.057353:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204878.057354:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204878.057357:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204878.057382:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204878.057386:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204878.057388:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204878.057390:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204878.057392:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788204878.057396:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788204878.057399:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057399:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204878.057401:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000040:2.0:1788204878.057404:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6343) 00000100:00000001:3.0:1788204878.057406:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204878.057409:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057412:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057419:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204878.057421:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057424:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204878.057426:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204878.057428:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204878.057433:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057440:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204878.057442:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057444:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204878.057445:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204878.057447:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.057450:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906688:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204878.057453:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204878.057455:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204878.057457:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057458:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204878.057461:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204878.057463:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204878.057467:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204878.057470:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204878.057473:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204878.057476:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906688, portal 10 00000100:00000001:3.0:1788204878.057477:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204878.057479:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788204878.057485:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057491:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204878.057493:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204878.057496:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204878.057498:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205906688, offset 0 00000400:00000200:3.0:1788204878.057502:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057505:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057509:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204878.057511:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204878.057513:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204878.057515:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204878.057516:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057518:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58939:58939:256:4294967295] 192.168.201.151@tcp LPNI seq info [58919:58919:8:4294967295] 00000400:00000200:3.0:1788204878.057520:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057525:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204878.057527:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057529:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204878.057532:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204878.057533:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204878.057536:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204878.057538:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204878.057540:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057541:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204878.057543:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204878.057545:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204878.057554:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204878.057555:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204878.057558:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204878.057562:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057569:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.057572:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057578:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057579:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204878.057583:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.057586:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204878.057589:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204878.057592:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301005467 00000100:00000001:3.0:1788204878.057593:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057595:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301005467 00000100:00000001:3.0:1788204878.057596:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057597:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204878.057598:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057600:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204878.057602:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204878.057606:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204878.057607:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204878.057608:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204878.057610:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204878.057611:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204878.057612:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204878.057614:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204878.057619:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057622:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204878.057623:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057624:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057625:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204878.057626:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057630:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204878.057632:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057649:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204878.057651:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057652:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057653:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057654:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204878.057655:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204878.057657:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.057659:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204878.057660:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204878.057663:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.057667:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906688:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204878.057669:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204878.057671:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057674:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204878.057675:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057676:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204878.057678:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204878.057679:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.057680:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204878.057682:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204878.057684:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204878.057684:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057685:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204878.057688:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204878.057689:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204878.057691:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204878.057693:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204878.057695:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204878.057696:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906816, portal 4 00000100:00000001:3.0:1788204878.057698:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204878.057698:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:3.0:1788204878.057702:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057706:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204878.057707:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204878.057708:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204878.057709:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205906816, offset 0 00000400:00000200:3.0:1788204878.057711:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057714:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057716:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204878.057717:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204878.057719:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204878.057720:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204878.057722:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057724:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58940:58940:256:4294967295] 192.168.201.151@tcp LPNI seq info [58920:58920:8:4294967295] 00000400:00000200:3.0:1788204878.057726:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057730:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204878.057733:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057734:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204878.057737:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204878.057738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204878.057741:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204878.057742:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204878.057744:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057746:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204878.057748:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204878.057750:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204878.057751:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204878.057752:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204878.057754:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204878.057756:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057761:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.057764:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057770:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057771:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204878.057775:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.057777:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204878.057780:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204878.057782:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301005353 00000100:00000001:3.0:1788204878.057784:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057786:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301005353 00000100:00000001:3.0:1788204878.057788:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057790:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204878.057792:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057794:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204878.057798:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204878.057805:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204878.057806:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204878.057808:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204878.057810:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204878.057811:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204878.057812:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204878.057814:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204878.057819:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057826:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204878.057827:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057828:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057829:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204878.057831:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057834:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204878.057836:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057841:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204878.057842:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057843:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057844:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057845:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204878.057846:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204878.057848:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.057849:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204878.057850:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204878.057852:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.057857:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204878.057859:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204878.057861:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057864:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204878.057865:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057867:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204878.057868:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204878.057869:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.057871:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906944:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204878.057872:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204878.057873:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204878.057874:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204878.057875:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204878.057877:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204878.057878:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204878.057880:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204878.057881:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204878.057883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204878.057885:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205906944, portal 4 00000100:00000001:3.0:1788204878.057886:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204878.057887:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204878.057890:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057893:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204878.057895:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204878.057896:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204878.057898:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205906944, offset 0 00000400:00000200:3.0:1788204878.057900:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057904:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204878.057906:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204878.057908:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204878.057909:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204878.057910:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204878.057912:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58941:58941:256:4294967295] 192.168.201.151@tcp LPNI seq info [58921:58921:8:4294967295] 00000400:00000200:3.0:1788204878.057914:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204878.057918:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204878.057921:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057923:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204878.057925:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204878.057926:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204878.057929:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204878.057931:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204878.057933:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204878.057934:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204878.057936:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204878.057938:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204878.057939:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204878.057940:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204878.057942:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204878.057943:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057948:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.057950:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057953:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057954:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204878.057956:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204878.057957:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204878.057959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204878.057960:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301005359 00000100:00000001:3.0:1788204878.057961:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057963:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301005359 00000100:00000001:3.0:1788204878.057964:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301009409 00000100:00000001:3.0:1788204878.057965:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204878.057966:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.057967:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204878.057969:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204878.057974:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204878.057975:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204878.057976:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204878.057977:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204878.057978:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204878.057979:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204878.057980:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204878.057984:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057988:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204878.057988:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.057989:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.057990:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204878.057992:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.057995:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204878.057997:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.058000:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204878.058001:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058002:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204878.058003:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058004:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204878.058005:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204878.058007:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.058008:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204878.058010:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204878.058012:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.058016:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205906944:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204878.058018:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058019:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.058021:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205906688/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.058024:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204878.058025:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204878.058026:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204878.058027:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204878.058028:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204878.058029:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204878.058030:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204878.058031:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:3.0:1788204878.058032:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204878.058033:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204878.058034:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204878.058035:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204878.058036:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058037:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.058038:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205906816/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.058042:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204878.058043:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204878.058044:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204878.058045:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204878.058045:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204878.058046:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204878.058047:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204878.058048:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:3.0:1788204878.058049:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204878.058050:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204878.058050:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788204878.058052:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204878.058053:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058053:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204878.058055:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205906944/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204878.058058:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204878.058059:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204878.058060:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204878.058061:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204878.058061:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204878.058062:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204878.058063:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204878.058063:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:3.0:1788204878.058064:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204878.058065:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204878.058066:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204878.058067:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204878.058067:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058068:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.058071:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204878.058071:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.058072:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204878.058073:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204878.248482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204878.248488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204878.248494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204878.248496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204878.248499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204878.248500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204878.248503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204878.248505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204878.248572:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204878.248574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204878.248576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204878.568479:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204878.568486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204878.568488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204878.568493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204878.568497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204878.568501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.568503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204878.568506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204878.568509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204879.016487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204879.016496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204879.016497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204879.016502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204879.016505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204879.016509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204879.016511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204879.016513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204879.016515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204879.080431:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204879.080436:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204879.080438:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204879.080443:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204879.080444:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204879.080450:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204879.080450:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204879.080455:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204879.080456:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204879.080460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788204879.080461:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204879.080463:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204879.080468:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204879.080469:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204879.080474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204879.080484:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204879.080487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204879.080490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204879.272520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204879.272526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204879.272530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204879.272531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204879.272533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204879.272534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204879.272536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204879.272537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204879.272551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204879.272553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204879.272556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204879.592447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204879.592454:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204879.592457:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204879.592463:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204879.592468:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204879.592472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204879.592475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204879.592478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204879.592481:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204880.040424:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204880.040430:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204880.040432:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204880.040436:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204880.040439:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204880.040442:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204880.040444:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204880.040446:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204880.040449:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.104471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204880.104479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204880.104481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204880.104487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204880.104490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204880.104494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204880.104497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204880.104497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204880.104499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204880.104500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204880.104501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.104502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204880.104504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204880.104506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204880.104508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204880.104510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204880.104511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204880.104513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296624:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204880.296631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296638:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204880.296641:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204880.296643:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204880.296647:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296650:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204880.296652:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296655:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296657:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296661:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204880.296667:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204880.296671:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204880.296673:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204880.296792:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204880.296797:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204880.296800:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204880.616476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204880.616483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204880.616485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204880.616489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204880.616492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204880.616495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204880.616497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204880.616500:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204880.616502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204881.064468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204881.064475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204881.064476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204881.064480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204881.064483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204881.064486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204881.064488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204881.064490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204881.064492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204881.128424:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204881.128431:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204881.128436:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204881.128437:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204881.128441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204881.128441:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204881.128448:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204881.128449:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204881.128454:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204881.128455:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204881.128460:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.128460:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204881.128463:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204881.128466:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204881.128468:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204881.128470:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204881.128473:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.128475:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204881.320514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204881.320520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204881.320524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204881.320525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204881.320527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204881.320528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204881.320530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204881.320531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204881.320544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204881.320546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204881.320548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204881.640480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204881.640487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204881.640488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204881.640492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204881.640495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204881.640500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.640503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204881.640506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.640509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204882.088479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204882.088485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204882.088487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204882.088491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204882.088493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204882.088496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204882.088498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204882.088499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204882.088502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204882.152477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204882.152481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204882.152485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204882.152486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204882.152490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204882.152490:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204882.152496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204882.152498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204882.152501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204882.152503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204882.152506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204882.152508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204882.152510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204882.152513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204882.152515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204882.152521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204882.152524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204882.152527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344468:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204882.344477:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204882.344480:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204882.344484:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344489:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204882.344492:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344494:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344496:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204882.344503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204882.344507:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204882.344508:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204882.344509:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344511:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204882.344512:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344513:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344514:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204882.344517:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204882.344518:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204882.344519:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344520:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204882.344521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204882.344523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204882.344595:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204882.344599:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204882.344601:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204882.664424:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204882.664433:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204882.664436:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204882.664443:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000040:0.0:1788204882.664445:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301009409, at 4301009920 00000100:00000040:1.0:1788204882.664447:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301009409, at 4301009920 00000100:00000001:3.0:1788204882.664448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204882.664453:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204882.664457:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204882.664459:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204882.664461:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788204882.664462:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204882.664465:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00000001:1.0:1788204883.112469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204883.112475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204883.112477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204883.112481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204883.112484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204883.112487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204883.112489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204883.112491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204883.112493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.176491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204883.176498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204883.176499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204883.176506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.176508:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204883.176514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204883.176516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788204883.176521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204883.176522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204883.176527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204883.176531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.176531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204883.176541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204883.176543:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204883.176559:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.176560:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204883.176565:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.176570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204883.176581:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204883.176585:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204883.176588:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204883.176590:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204883.176593:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204883.176603:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6338 00000400:00000200:0.0:1788204883.176611:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204883.176616:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204883.176618:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204883.176621:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176623:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.176625:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176627:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204883.176629:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204883.176631:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204883.176633:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204883.176636:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204883.176637:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204883.176639:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204883.176641:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176642:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.176643:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176645:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204883.176647:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204883.176649:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176651:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204883.176653:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204883.176656:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204883.176658:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.176660:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204883.176661:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204883.176669:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204883.176674:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204883.176678:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204883.176680:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.176683:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204883.176686:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204883.176690:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204883.176701:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788204883.176708:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204883.176719:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204883.176728:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204883.176731:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204883.176735:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204883.176755:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204883.176757:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204883.176760:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204883.176761:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204883.176762:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204883.176764:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204883.176767:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6338 00000400:00000200:0.0:1788204883.176770:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204883.176773:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204883.176775:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204883.176778:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204883.176779:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204883.176781:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.176786:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204883.176786:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204883.176793:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204883.176794:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204883.176800:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204883.176801:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204883.176803:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:3.0:1788204883.176807:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204883.176813:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204883.176822:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204883.176824:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204883.176828:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204883.176829:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.176841:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1788204883.176842:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176846:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788204883.176846:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204883.176853:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204883.176854:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.176865:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788204883.176867:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205907072:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788204883.176871:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204883.176875:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788204883.176879:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204883.176880:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204883.176881:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.176883:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.176887:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204883.176889:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1788204883.176895:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:0.0:1788204883.176897:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204883.176899:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204883.176908:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.176914:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204883.176916:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:3.0:1788204883.176919:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000008:00100000:0.0:1788204883.176923:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788204883.176928:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000008:00000001:0.0:1788204883.176930:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.176934:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204883.176936:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907072, portal 25 00000100:00000010:0.0:1788204883.176938:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204883.176941:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1788204883.176943:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:3.0:1788204883.176943:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 02000000:00000001:0.0:1788204883.176954:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204883.176955:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.176959:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204883.176960:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204883.176964:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204883.176969:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204883.176985:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:3.0:1788204883.176988:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204883.176995:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.177000:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00100000:0.0:1788204883.177005:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204883.177005:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204883.177017:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205907072, offset 0 00000100:00000001:0.0:1788204883.177020:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204883.177024:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.177034:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204883.177041:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204883.177049:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204883.177063:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.177063:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:0.0:1788204883.177068:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204883.177081:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204883.177083:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204883.177084:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1788204883.177091:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204883.177096:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:0.0:1788204883.177098:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204883.177103:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6338 00000400:00000200:3.0:1788204883.177103:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204883.177110:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204883.177114:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.177119:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58942:58942:256:4294967295] 192.168.201.151@tcp LPNI seq info [58922:58922:8:4294967295] 00000400:00000200:0.0:1788204883.177121:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204883.177125:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204883.177134:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177136:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.177139:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177143:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204883.177145:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788204883.177151:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204883.177152:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204883.177155:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204883.177158:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204883.177167:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204883.177168:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:0.0:1788204883.177175:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204883.177185:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788204883.177186:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177193:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1788204883.177197:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:0.0:1788204883.177199:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177206:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204883.177210:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.0:1788204883.177214:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:0.0:1788204883.177220:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.0:1788204883.177220:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1788204883.177231:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204883.177234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204883.177235:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788204883.177240:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 02000000:00000001:0.0:1788204883.177241:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204883.177247:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204883.177248:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177255:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:3.0:1788204883.177256:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000008:00100000:0.0:1788204883.177259:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204883.177262:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.177264:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.177264:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:0.0:1788204883.177279:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000200:3.0:1788204883.177286:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000040:0.0:1788204883.177288:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788204883.177291:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 02000000:00000001:0.0:1788204883.177292:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204883.177296:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.177299:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788204883.177299:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 02000000:00000001:0.0:1788204883.177305:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204883.177309:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1788204883.177310:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204883.177321:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000200:3.0:1788204883.177323:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204883.177330:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177345:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1788204883.177347:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204883.177354:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204883.177398:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204883.177399:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204883.177410:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.177421:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177433:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204883.177435:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204883.177438:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204883.177441:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204883.177443:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204883.177443:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204883.177446:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204883.177452:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204883.177453:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204883.177461:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6338 00000100:00000001:3.0:1788204883.177461:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301014530 00000400:00000200:0.0:1788204883.177472:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:3.0:1788204883.177474:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c60. 00000400:00000200:0.0:1788204883.177486:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204883.177489:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204883.177489:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788204883.177496:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177499:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.177501:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204883.177501:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301014530 00000100:00000001:0.0:1788204883.177511:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000010:3.0:1788204883.177512:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4420. 00000020:00000001:0.0:1788204883.177523:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204883.177526:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1788204883.177527:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204883.177530:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204883.177533:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788204883.177535:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788204883.177536:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204883.177541:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204883.177557:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177562:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788204883.177565:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177568:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204883.177571:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177574:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788204883.177574:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204883.177581:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204883.177592:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204883.177601:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204883.177605:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1788204883.177606:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204883.177608:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788204883.177609:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1788204883.177611:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 02000000:00000001:0.0:1788204883.177612:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.177618:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204883.177618:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204883.177624:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000008:00100000:0.0:1788204883.177625:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204883.177634:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177635:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788204883.177643:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204883.177651:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000200:3.0:1788204883.177652:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204883.177658:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204883.177665:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204883.177667:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204883.177674:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204883.177677:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204883.177681:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204883.177687:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204883.177688:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788204883.177698:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177701:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204883.177704:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177708:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1788204883.177709:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204883.177711:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204883.177718:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204883.177726:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204883.177733:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204883.177748:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.177757:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204883.177771:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.177774:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204883.177777:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6348) 00000100:00000001:3.0:1788204883.177791:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204883.177796:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177797:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.177799:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177802:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204883.177806:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204883.177811:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.177816:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204883.177819:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204883.177824:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.177833:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205907072:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204883.177838:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177840:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.177843:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205907072/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.177849:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204883.177853:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204883.177856:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204883.177860:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204883.177862:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204883.177864:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204883.177867:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204883.177869:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:3.0:1788204883.177872:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204883.177874:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204883.177876:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204883.177878:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204883.177880:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177881:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.177888:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204883.177889:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.177891:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204883.177892:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204883.177894:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204883.177897:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.177902:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204883.177903:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.177905:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204883.177906:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204883.177908:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204883.177909:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907200:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204883.177911:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204883.177913:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204883.177914:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.177915:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204883.177918:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204883.177919:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.177922:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204883.177924:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204883.177927:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204883.177929:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907200, portal 10 00000100:00000001:3.0:1788204883.177930:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204883.177931:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204883.177937:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.177941:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204883.177942:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204883.177944:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204883.177946:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205907200, offset 0 00000400:00000200:3.0:1788204883.177948:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204883.177951:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.177953:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204883.177955:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204883.177957:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204883.177959:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204883.177960:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204883.177963:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58943:58943:256:4294967295] 192.168.201.151@tcp LPNI seq info [58923:58923:8:4294967295] 00000400:00000200:3.0:1788204883.177965:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.177970:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204883.177973:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.177975:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204883.177978:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204883.177980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204883.177983:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204883.178000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204883.178003:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.178005:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204883.178007:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204883.178009:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204883.178011:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204883.178012:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204883.178014:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204883.178016:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178022:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178023:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178028:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178028:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204883.178031:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.178034:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204883.178036:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204883.178038:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301010587 00000100:00000001:3.0:1788204883.178040:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178041:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301010587 00000100:00000001:3.0:1788204883.178043:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178044:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204883.178045:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178047:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204883.178049:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204883.178054:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204883.178055:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204883.178055:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204883.178057:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204883.178058:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204883.178059:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204883.178061:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204883.178066:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178069:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204883.178070:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178071:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178072:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204883.178073:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178077:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204883.178079:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178083:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204883.178084:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178085:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178086:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178088:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204883.178089:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204883.178091:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.178093:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204883.178095:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204883.178097:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.178101:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907200:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204883.178103:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204883.178106:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178109:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204883.178110:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.178111:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204883.178112:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204883.178114:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204883.178116:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204883.178119:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204883.178120:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204883.178121:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.178123:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204883.178125:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204883.178127:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.178130:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204883.178133:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204883.178136:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204883.178139:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907328, portal 4 00000100:00000001:3.0:1788204883.178141:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204883.178142:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204883.178148:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178155:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204883.178158:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204883.178160:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204883.178163:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205907328, offset 0 00000400:00000200:3.0:1788204883.178166:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204883.178170:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.178173:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204883.178176:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204883.178178:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204883.178181:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204883.178183:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204883.178186:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58944:58944:256:4294967295] 192.168.201.151@tcp LPNI seq info [58924:58924:8:4294967295] 00000400:00000200:3.0:1788204883.178190:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.178197:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204883.178201:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.178204:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204883.178208:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204883.178211:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204883.178215:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204883.178217:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204883.178221:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.178222:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204883.178225:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204883.178228:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204883.178231:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204883.178233:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204883.178236:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204883.178239:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178248:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178251:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178258:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178259:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204883.178262:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.178265:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204883.178267:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204883.178268:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301010473 00000100:00000001:3.0:1788204883.178270:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178272:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301010473 00000100:00000001:3.0:1788204883.178273:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178274:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204883.178275:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178276:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204883.178279:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204883.178283:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204883.178284:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204883.178285:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204883.178287:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204883.178288:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204883.178289:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204883.178291:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204883.178296:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178299:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204883.178301:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178302:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178303:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204883.178305:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178308:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204883.178310:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178315:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204883.178317:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178318:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178318:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178319:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204883.178321:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204883.178323:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.178324:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204883.178326:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204883.178328:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.178332:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204883.178334:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204883.178336:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178340:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204883.178342:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.178343:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204883.178344:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204883.178345:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204883.178347:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907456:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204883.178349:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204883.178350:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204883.178351:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204883.178352:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204883.178355:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204883.178356:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204883.178358:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204883.178360:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204883.178362:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204883.178379:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907456, portal 4 00000100:00000001:3.0:1788204883.178381:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204883.178382:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204883.178386:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178390:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204883.178392:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204883.178394:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204883.178395:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205907456, offset 0 00000400:00000200:3.0:1788204883.178397:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204883.178400:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.178402:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204883.178404:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204883.178406:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204883.178408:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204883.178409:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204883.178411:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58945:58945:256:4294967295] 192.168.201.151@tcp LPNI seq info [58925:58925:8:4294967295] 00000400:00000200:3.0:1788204883.178413:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204883.178417:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204883.178419:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.178421:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204883.178424:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204883.178425:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204883.178428:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204883.178430:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204883.178432:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204883.178434:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204883.178436:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204883.178438:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204883.178439:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204883.178441:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204883.178442:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204883.178445:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178450:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178451:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204908 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178455:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178456:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204883.178458:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204883.178459:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204883.178461:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204883.178462:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301010480 00000100:00000001:3.0:1788204883.178464:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178465:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301010480 00000100:00000001:3.0:1788204883.178466:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301014530 00000100:00000001:3.0:1788204883.178468:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204883.178469:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178470:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204883.178472:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204883.178476:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204883.178476:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204883.178477:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204883.178478:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204883.178479:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204883.178481:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204883.178483:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204883.178489:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178500:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204883.178501:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178503:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178504:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204883.178508:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178515:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204883.178519:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178526:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204883.178529:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178530:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204883.178531:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178533:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204883.178535:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204883.178538:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.178541:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204883.178542:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204883.178558:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.178563:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907456:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204883.178566:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178567:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178569:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205907200/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178573:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204883.178574:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204883.178576:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204883.178578:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204883.178579:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204883.178580:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204883.178581:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204883.178582:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204883.178584:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204883.178585:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204883.178586:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204883.178588:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204883.178589:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178590:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178592:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205907328/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178595:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204883.178596:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204883.178596:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204883.178597:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204883.178598:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204883.178599:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204883.178600:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204883.178601:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788204883.178604:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204883.178604:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204883.178605:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204883.178607:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204883.178607:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178608:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204883.178610:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205907456/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204883.178613:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204883.178613:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204883.178614:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204883.178615:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204883.178616:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204883.178616:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204883.178618:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204883.178619:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788204883.178620:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204883.178621:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204883.178622:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204883.178623:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204883.178624:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178625:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.178627:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204883.178628:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.178629:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204883.178630:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204883.368580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204883.368590:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204883.368597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204883.368599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204883.368603:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204883.368604:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204883.368609:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204883.368610:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204883.368634:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204883.368638:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204883.368642:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204883.688508:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204883.688519:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204883.688522:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204883.688529:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204883.688533:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204883.688548:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.688552:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204883.688555:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204883.688559:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204884.136468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204884.136475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204884.136477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204884.136482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204884.136486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204884.136490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204884.136492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204884.136494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204884.136497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.200497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204884.200506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204884.200508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.200514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204884.200518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204884.200523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.200525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204884.200537:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.200541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204884.200583:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204884.200590:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204884.200593:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204884.200598:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204884.200602:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204884.200604:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204884.200606:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204884.200608:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204884.200610:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204884.392657:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204884.392666:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204884.392674:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204884.392675:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204884.392679:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204884.392680:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204884.392684:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204884.392685:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204884.392748:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204884.392753:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204884.392757:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204884.712480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204884.712489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204884.712491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.712499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204884.712503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204884.712508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.712511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204884.712515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.712544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204885.160504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204885.160535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204885.160537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204885.160542:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204885.160545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204885.160550:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204885.160553:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204885.160555:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204885.160557:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204885.224486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204885.224486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204885.224496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204885.224497:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204885.224500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.224500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204885.224508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204885.224508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204885.224526:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204885.224527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204885.224530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204885.224531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204885.224537:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204885.224539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204885.224542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.224542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204885.224544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204885.224547:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204885.416562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204885.416569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204885.416574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204885.416575:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204885.416577:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204885.416578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204885.416581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204885.416581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204885.416599:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204885.416601:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204885.416604:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204885.736528:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204885.736537:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204885.736539:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204885.736544:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204885.736548:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204885.736552:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204885.736554:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204885.736556:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204885.736559:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204886.184432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204886.184440:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204886.184442:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204886.184454:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204886.184457:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204886.184461:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204886.184463:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204886.184467:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204886.184471:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204886.248490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204886.248490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204886.248518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204886.248520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204886.248522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204886.248523:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204886.248532:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204886.248536:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204886.248537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204886.248541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204886.248545:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.248546:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204886.248550:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204886.248553:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.248556:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204886.248557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204886.248561:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.248565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204886.440502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204886.440513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204886.440522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204886.440524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204886.440527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204886.440529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204886.440533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204886.440534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204886.440591:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204886.440595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204886.440599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204886.760495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204886.760505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204886.760507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204886.760514:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204886.760517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204886.760520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.760523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204886.760525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.760528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204887.208538:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204887.208548:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204887.208551:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204887.208558:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204887.208563:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204887.208568:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204887.208571:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204887.208574:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204887.208577:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204887.272517:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204887.272517:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204887.272529:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204887.272529:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204887.272533:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204887.272533:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204887.272542:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788204887.272542:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204887.272548:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204887.272554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.272558:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204887.272559:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204887.272564:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.272565:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204887.272568:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204887.272569:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204887.272573:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.272576:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204887.464546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204887.464553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204887.464563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204887.464564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204887.464567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204887.464568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204887.464571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204887.464572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204887.464588:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204887.464591:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204887.464593:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204887.784454:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204887.784456:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301014530, at 4301015040 00000100:00000001:3.0:1788204887.784462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788204887.784464:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301014530, at 4301015040 00000100:00000001:3.0:1788204887.784478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204887.784482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204887.784485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788204887.784490:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000001:3.0:1788204887.784490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1788204887.784494:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000001:3.0:1788204887.784495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204887.784499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.784502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204888.232428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204888.232436:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204888.232438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204888.232450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204888.232456:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204888.232481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204888.232483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204888.232486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204888.232489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.296436:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204888.296447:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204888.296447:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204888.296449:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204888.296453:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204888.296466:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204888.296467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204888.296468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204888.296472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204888.296474:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204888.296475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204888.296475:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204888.296479:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204888.296480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.296480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204888.296484:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204888.296486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204888.296489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204888.296489:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204888.296493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204888.296494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.296495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204888.296499:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6343 00000100:00000001:3.0:1788204888.296499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204888.296512:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204888.296521:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204888.296525:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204888.296528:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296532:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.296534:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296536:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204888.296540:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204888.296548:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204888.296552:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204888.296556:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204888.296558:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204888.296561:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204888.296564:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296566:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.296568:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296570:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204888.296574:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204888.296577:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296580:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204888.296583:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204888.296587:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204888.296591:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296593:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204888.296596:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204888.296611:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204888.296614:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204888.296618:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204888.296620:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296623:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204888.296625:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204888.296629:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204888.296640:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204888.296646:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204888.296655:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204888.296664:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204888.296668:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204888.296672:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204888.296696:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204888.296698:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204888.296701:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204888.296702:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204888.296703:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204888.296705:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204888.296707:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6343 00000400:00000200:2.0:1788204888.296710:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204888.296712:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204888.296714:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204888.296716:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296718:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.296719:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296720:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204888.296722:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204888.296723:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204888.296724:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204888.296725:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204888.296726:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204888.296727:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204888.296729:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296730:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.296731:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296732:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204888.296734:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204888.296736:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296738:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204888.296740:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204888.296745:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204888.296747:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296749:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204888.296749:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204888.296754:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788204888.296755:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204888.296758:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.296758:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204888.296762:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204888.296768:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204888.296769:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204888.296772:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204888.296777:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204888.296778:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296782:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204888.296785:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204888.296788:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.296788:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.296791:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204888.296795:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204888.296797:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204888.296800:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.296802:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204888.296805:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204888.296809:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204888.296811:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205907584:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204888.296819:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204888.296824:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788204888.296825:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788204888.296827:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.296829:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204888.296833:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204888.296833:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204888.296840:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 00000100:00000001:2.0:1788204888.296841:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.296843:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204888.296845:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204888.296848:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204888.296849:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204888.296852:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204888.296852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204888.296857:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788204888.296858:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00080000:2.0:1788204888.296860:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6343 00000400:00000200:2.0:1788204888.296865:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204888.296867:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:3.0:1788204888.296867:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00080000:2.0:1788204888.296869:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788204888.296875:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907584, portal 25 00000100:00000040:2.0:1788204888.296878:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204888.296879:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204888.296882:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204888.296882:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:2.0:1788204888.296887:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296890:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:3.0:1788204888.296891:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204888.296892:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204888.296894:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204888.296896:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204888.296899:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204888.296902:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:3.0:1788204888.296904:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788204888.296905:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788204888.296910:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204888.296911:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296915:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1788204888.296915:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000040:2.0:1788204888.296918:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296921:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:3.0:1788204888.296921:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205907584, offset 0 00000020:00000040:2.0:1788204888.296926:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204888.296928:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204888.296932:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296933:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204888.296936:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204888.296938:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204888.296939:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296940:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204888.296942:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204888.296943:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:2.0:1788204888.296946:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296948:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204888.296952:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000400:00000200:3.0:1788204888.296953:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:2.0:1788204888.296956:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204888.296959:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788204888.296960:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1788204888.296961:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.296964:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204888.296965:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204888.296966:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204888.296967:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000010:2.0:1788204888.296970:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000400:00000200:3.0:1788204888.296973:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788204888.296975:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204888.296979:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204888.296987:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204888.296988:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.296989:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204888.296995:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204888.296996:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58946:58946:256:4294967295] 192.168.201.151@tcp LPNI seq info [58926:58926:8:4294967295] 00000400:00000200:3.0:1788204888.297008:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204888.297015:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204888.297020:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204888.297020:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788204888.297024:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204888.297027:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204888.297029:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.0:1788204888.297032:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204888.297033:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204888.297037:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6343 00000800:00000010:3.0:1788204888.297039:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000400:00000200:2.0:1788204888.297040:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204888.297045:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204888.297046:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204888.297050:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788204888.297051:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204888.297052:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.297056:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.297058:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788204888.297058:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788204888.297061:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204888.297063:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204888.297064:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204888.297068:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000200:3.1:1788204888.297068:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204888.297072:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788204888.297073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788204888.297074:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000100:3.0:1788204888.297078:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297081:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000040:2.0:1788204888.297084:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204888.297086:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204888.297088:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297089:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204888.297090:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204888.297093:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204888.297096:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297098:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788204888.297099:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204888.297102:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204888.297104:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204888.297105:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 02000000:00000001:2.0:1788204888.297106:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204888.297108:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:2.0:1788204888.297109:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204888.297111:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204888.297114:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297114:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204888.297115:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204888.297119:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204888.297120:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204888.297124:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204888.297128:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204888.297130:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204888.297133:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204888.297135:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204888.297135:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788204888.297139:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204888.297141:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204888.297142:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204888.297147:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297153:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204888.297154:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204888.297156:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204888.297161:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204888.297163:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297163:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204888.297166:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204888.297171:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204888.297176:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301019649 00000100:00000001:2.0:1788204888.297178:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204888.297181:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4360. 00000100:00000040:2.0:1788204888.297183:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6353) 00000100:00000001:3.0:1788204888.297190:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204888.297197:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301019649 00000100:00000010:3.0:1788204888.297200:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4720. 00000100:00000001:3.0:1788204888.297203:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204888.297205:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204888.297207:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297209:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204888.297212:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204888/real 1788204888] req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204888.297218:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204888.297219:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204888.297221:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204888.297223:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204888.297224:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204888.297226:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204888.297228:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204888.297233:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297237:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204888.297239:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297240:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297242:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204888.297244:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297248:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204888.297251:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297254:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204888.297256:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297257:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297258:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297259:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204888.297261:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204888.297263:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204888.297265:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204888.297266:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204888.297269:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204888.297274:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205907584:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204888.297278:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297280:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.297282:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205907584/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297286:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204888.297288:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204888.297290:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204888.297292:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204888.297293:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204888.297294:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204888.297295:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204888.297297:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788204888.297300:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204888.297301:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204888.297303:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788204888.297307:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204888.297308:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297311:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297319:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204888.297321:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297323:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204888.297325:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204888.297326:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204888.297330:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297337:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.297338:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297340:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204888.297342:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204888.297344:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204888.297347:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907712:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204888.297350:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204888.297351:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204888.297353:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297355:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204888.297357:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204888.297358:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204888.297361:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204888.297379:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204888.297382:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204888.297384:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907712, portal 10 00000100:00000001:3.0:1788204888.297386:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204888.297386:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788204888.297392:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297396:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204888.297398:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204888.297399:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204888.297400:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205907712, offset 0 00000400:00000200:3.0:1788204888.297403:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297405:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297407:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204888.297409:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204888.297411:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204888.297412:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204888.297414:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297416:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58947:58947:256:4294967295] 192.168.201.151@tcp LPNI seq info [58927:58927:8:4294967295] 00000400:00000200:3.0:1788204888.297418:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297423:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204888.297426:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297428:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204888.297431:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204888.297433:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204888.297435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204888.297437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204888.297439:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297440:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204888.297443:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204888.297445:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204888.297448:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204888.297450:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204888.297452:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204888.297455:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297474:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.297478:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297485:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297486:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204888.297490:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204888.297495:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204888.297498:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204888.297500:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301015707 00000100:00000001:3.0:1788204888.297503:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297504:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301015707 00000100:00000001:3.0:1788204888.297506:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297507:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204888.297509:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297510:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204888.297512:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204888/real 1788204888] req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204888.297517:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204888.297518:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204888.297519:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204888.297520:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204888.297521:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204888.297523:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204888.297525:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204888.297530:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297534:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204888.297535:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297536:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297537:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204888.297538:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297542:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204888.297544:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297548:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204888.297549:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297550:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297551:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297552:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204888.297554:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204888.297555:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204888.297557:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204888.297558:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204888.297560:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204888.297564:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907712:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204888.297567:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204888.297569:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297577:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.297578:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297579:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204888.297581:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204888.297582:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204888.297583:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204888.297585:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204888.297586:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204888.297587:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297588:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204888.297590:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204888.297591:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204888.297593:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204888.297594:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204888.297597:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204888.297598:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907840, portal 4 00000100:00000001:3.0:1788204888.297599:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204888.297601:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:3.0:1788204888.297605:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297608:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204888.297610:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204888.297611:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204888.297612:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205907840, offset 0 00000400:00000200:3.0:1788204888.297614:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297616:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297618:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204888.297620:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204888.297622:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204888.297623:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204888.297625:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297627:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58948:58948:256:4294967295] 192.168.201.151@tcp LPNI seq info [58928:58928:8:4294967295] 00000400:00000200:3.0:1788204888.297630:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297641:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204888.297644:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297646:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204888.297648:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204888.297650:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204888.297653:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204888.297654:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204888.297657:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297658:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204888.297660:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204888.297662:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204888.297664:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204888.297665:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204888.297667:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204888.297669:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297674:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.297675:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297679:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297680:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204888.297682:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204888.297684:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204888.297686:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204888.297687:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301015593 00000100:00000001:3.0:1788204888.297733:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297734:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301015593 00000100:00000001:3.0:1788204888.297736:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297738:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204888.297740:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297741:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204888.297744:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204888/real 1788204888] req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204888.297749:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204888.297750:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204888.297751:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204888.297753:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204888.297755:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204888.297756:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204888.297759:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204888.297767:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297774:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204888.297775:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297777:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297778:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204888.297782:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297789:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204888.297793:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297800:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204888.297802:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297803:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297804:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297805:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204888.297807:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204888.297808:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204888.297811:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204888.297813:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204888.297815:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204888.297820:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204888.297822:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204888.297824:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297827:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.297828:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297830:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204888.297831:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204888.297832:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204888.297834:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907968:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204888.297835:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204888.297836:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204888.297837:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204888.297838:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204888.297840:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204888.297841:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204888.297843:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204888.297845:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204888.297847:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204888.297849:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205907968, portal 4 00000100:00000001:3.0:1788204888.297850:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204888.297851:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204888.297855:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297859:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204888.297861:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204888.297862:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204888.297864:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205907968, offset 0 00000400:00000200:3.0:1788204888.297866:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297869:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297871:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204888.297873:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204888.297874:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204888.297876:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204888.297877:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204888.297879:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58949:58949:256:4294967295] 192.168.201.151@tcp LPNI seq info [58929:58929:8:4294967295] 00000400:00000200:3.0:1788204888.297882:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204888.297886:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204888.297889:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297891:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204888.297893:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204888.297895:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204888.297897:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204888.297899:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204888.297901:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204888.297902:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204888.297904:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204888.297906:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204888.297908:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204888.297909:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204888.297911:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204888.297913:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297917:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.297919:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204913 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297923:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297924:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204888.297926:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204888.297928:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204888.297930:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204888.297931:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301015599 00000100:00000001:3.0:1788204888.297933:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297935:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301015599 00000100:00000001:3.0:1788204888.297936:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301019649 00000100:00000001:3.0:1788204888.297937:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204888.297938:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.297939:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204888.297941:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204888/real 1788204888] req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204888.297945:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204888.297945:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204888.297947:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204888.297948:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204888.297949:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204888.297950:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204888.297952:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204888.297956:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297959:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204888.297960:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297961:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297962:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204888.297963:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297968:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204888.297970:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297975:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204888.297976:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297977:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204888.297978:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297979:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204888.297980:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204888.297981:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204888.297983:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204888.297984:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204888.297986:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204888.297990:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205907968:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204888.297991:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.297992:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.297994:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205907712/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.297997:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204888.297998:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204888.297999:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204888.298000:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204888.298001:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204888.298002:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204888.298003:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204888.298004:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204888.298005:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204888.298006:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204888.298007:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788204888.298008:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204888.298009:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.298010:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.298012:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205907840/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.298016:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204888.298017:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204888.298018:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204888.298019:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204888.298020:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204888.298020:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204888.298021:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204888.298022:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788204888.298024:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204888.298024:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204888.298025:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788204888.298026:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204888.298027:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.298027:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204888.298029:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205907968/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204913 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204888.298032:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204888.298033:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204888.298033:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204888.298034:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204888.298035:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204888.298036:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204888.298036:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204888.298037:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:3.0:1788204888.298038:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204888.298039:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204888.298039:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204888.298040:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204888.298041:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.298042:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.298044:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204888.298045:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.298046:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204888.298047:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204888.488488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204888.488495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204888.488501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204888.488502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204888.488505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204888.488506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204888.488509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204888.488510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204888.488567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204888.488573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204888.488577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204888.808497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204888.808505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204888.808507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204888.808513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204888.808516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204888.808520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.808523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.808525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204888.808528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204889.256532:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204889.256541:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204889.256543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204889.256554:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204889.256559:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204889.256564:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204889.256567:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204889.256577:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204889.256584:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.320493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204889.320502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204889.320504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.320510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204889.320514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204889.320517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.320519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.320521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.320524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204889.320540:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204889.320546:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204889.320547:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204889.320551:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204889.320553:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204889.320556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204889.320557:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204889.320559:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204889.320561:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204889.512554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204889.512562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204889.512568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204889.512569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204889.512571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204889.512572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204889.512575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204889.512576:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204889.512592:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204889.512595:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204889.512598:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204889.832463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204889.832473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204889.832476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.832482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204889.832487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204889.832492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.832495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.832498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.832501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204890.280530:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204890.280542:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204890.280545:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204890.280552:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204890.280555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204890.280560:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204890.280562:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204890.280565:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204890.280567:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204890.344459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204890.344459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204890.344470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204890.344473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.344476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788204890.344479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204890.344479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204890.344485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204890.344485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204890.344490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204890.344491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204890.344493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204890.344495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204890.344498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204890.344501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.344502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204890.344505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204890.344507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204890.536477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204890.536487:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204890.536489:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204890.536492:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536495:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204890.536498:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536500:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536503:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204890.536507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204890.536511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204890.536512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204890.536569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204890.536573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204890.536577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204890.856524:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204890.856531:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204890.856534:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204890.856539:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204890.856543:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204890.856546:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204890.856549:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204890.856552:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204890.856555:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204891.304480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204891.304486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204891.304488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204891.304491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204891.304494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204891.304497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204891.304499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204891.304501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204891.304503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204891.368469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204891.368475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204891.368477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204891.368482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204891.368484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204891.368485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204891.368489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204891.368490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204891.368492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204891.368494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788204891.368497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204891.368498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204891.368502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204891.368503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204891.368505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204891.368508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204891.368510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204891.368513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204891.560519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204891.560525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204891.560529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204891.560536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204891.560539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204891.560540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204891.560543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204891.560543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204891.560563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204891.560565:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204891.560568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204891.880474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204891.880483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204891.880486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204891.880492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204891.880495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204891.880499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204891.880502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204891.880505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204891.880509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204892.328527:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204892.328536:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204892.328539:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204892.328544:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204892.328548:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204892.328551:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204892.328554:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204892.328557:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204892.328560:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204892.392473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204892.392479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204892.392480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204892.392484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204892.392485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204892.392489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204892.392490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204892.392496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204892.392497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204892.392499:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204892.392504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204892.392505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204892.392506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204892.392510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204892.392511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204892.392515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204892.392517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204892.392520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204892.584487:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204892.584490:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204892.584494:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584499:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204892.584501:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584503:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584505:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204892.584512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204892.584516:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204892.584518:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204892.584519:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584521:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204892.584523:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584524:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584526:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204892.584531:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204892.584533:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204892.584534:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584537:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204892.584538:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584540:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204892.584542:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204892.584608:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204892.584613:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204892.584616:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204892.904469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204892.904470:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301019649, at 4301020160 00000100:00000040:0.0:1788204892.904471:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301019649, at 4301020160 00000100:00000001:3.0:1788204892.904475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204892.904479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204892.904483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204892.904486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204892.904487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000010:0.0:1788204892.904489:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000001:3.0:1788204892.904489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204892.904494:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204892.904496:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204892.904499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204893.352494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204893.352501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204893.352504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204893.352510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204893.352514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204893.352528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204893.352531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204893.352533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204893.352536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204893.416491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204893.416492:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204893.416499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204893.416504:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204893.416504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416508:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788204893.416511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788204893.416514:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204893.416517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204893.416520:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204893.416523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204893.416527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204893.416530:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6348 00000100:00000001:2.0:1788204893.416533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204893.416539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204893.416543:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204893.416551:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204893.416555:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204893.416560:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416564:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.416567:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204893.416570:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204893.416571:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204893.416573:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1788204893.416576:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204893.416576:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204893.416578:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00100000:2.0:1788204893.416580:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000020:00000001:0.0:1788204893.416582:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788204893.416583:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788204893.416586:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:2.0:1788204893.416586:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204893.416592:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788204893.416593:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204893.416595:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204893.416596:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788204893.416598:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204893.416602:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416604:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.416606:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416607:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204893.416610:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204893.416613:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416615:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204893.416617:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204893.416620:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204893.416622:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416623:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204893.416625:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204893.416633:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204893.416635:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204893.416638:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204893.416640:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416642:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204893.416644:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204893.416647:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204893.416655:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204893.416661:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204893.416668:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204893.416676:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204893.416679:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204893.416683:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204893.416699:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204893.416701:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204893.416703:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204893.416705:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204893.416706:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204893.416708:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204893.416709:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6348 00000400:00000200:0.0:1788204893.416713:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204893.416715:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204893.416717:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204893.416720:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416722:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.416724:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416726:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204893.416728:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204893.416728:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1788204893.416729:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204893.416732:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204893.416734:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204893.416734:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204893.416739:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204893.416739:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204893.416743:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204893.416748:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416750:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204893.416751:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204893.416752:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416755:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204893.416759:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204893.416764:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416769:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204893.416772:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204893.416777:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788204893.416778:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204893.416779:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204893.416782:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416784:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204893.416786:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1788204893.416788:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204893.416790:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788204893.416791:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416795:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.416798:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204893.416800:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204893.416803:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204893.416806:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788204893.416806:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205908096:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204893.416809:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416813:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204893.416815:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204893.416816:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204893.416817:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204893.416822:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788204893.416823:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204893.416825:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.416827:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204893.416832:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204893.416833:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204893.416841:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:3.0:1788204893.416841:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000001:0.0:1788204893.416844:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.416846:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204893.416849:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204893.416857:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204893.416860:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204893.416863:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204893.416864:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00080000:0.0:1788204893.416865:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204893.416870:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204893.416871:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204893.416873:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788204893.416875:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00080000:0.0:1788204893.416876:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6348 00000400:00000200:0.0:1788204893.416882:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204893.416884:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000200:3.0:1788204893.416885:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908096, portal 25 00000100:00080000:0.0:1788204893.416886:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204893.416891:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204893.416891:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204893.416895:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204893.416895:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:0.0:1788204893.416899:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416902:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204893.416904:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204893.416905:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204893.416907:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:3.0:1788204893.416908:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204893.416909:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204893.416912:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204893.416916:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204893.416924:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416928:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.416932:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416933:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204893.416933:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1788204893.416936:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204893.416938:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204893.416939:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204893.416940:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204893.416942:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204893.416943:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416944:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204893.416946:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204893.416947:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416949:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204893.416951:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204893.416954:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204893.416958:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:3.0:1788204893.416959:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204893.416961:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.416965:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204893.416966:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788204893.416966:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 02000000:00000001:0.0:1788204893.416970:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204893.416974:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205908096, offset 0 02000000:00000010:0.0:1788204893.416977:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204893.416982:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.416986:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00100000:0.0:1788204893.416991:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204893.417003:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204893.417004:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204893.417008:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417015:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788204893.417017:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204893.417024:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204893.417032:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417033:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:0.0:1788204893.417038:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204893.417041:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1788204893.417046:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788204893.417049:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417050:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204893.417054:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204893.417057:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58950:58950:256:4294967295] 192.168.201.151@tcp LPNI seq info [58930:58930:8:4294967295] 00000100:00000001:0.0:1788204893.417059:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204893.417065:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6348 00000400:00000200:3.0:1788204893.417068:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204893.417072:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204893.417076:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204893.417078:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204893.417080:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417082:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.417083:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417085:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204893.417090:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204893.417093:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204893.417096:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204893.417096:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:0.0:1788204893.417100:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204893.417102:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204893.417104:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204893.417106:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417107:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204893.417110:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417112:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204893.417115:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000800:00000200:3.0:1788204893.417117:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204893.417119:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204893.417124:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204893.417125:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1788204893.417127:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 02000000:00000001:0.0:1788204893.417128:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204893.417133:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204893.417134:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204893.417136:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204893.417141:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204893.417141:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204893.417146:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204893.417149:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000100:00000010:0.0:1788204893.417150:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204893.417155:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204893.417158:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204893.417161:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204893.417161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204893.417165:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204893.417168:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788204893.417169:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 02000000:00000001:0.0:1788204893.417170:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204893.417174:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204893.417180:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204893.417180:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.417183:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00100000:0.0:1788204893.417188:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204893.417193:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204893.417194:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204893.417198:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204893.417204:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204893.417206:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788204893.417216:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417217:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000100:00000040:0.0:1788204893.417220:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6358) 00000400:00000010:3.0:1788204893.417222:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204893.417228:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204893.417233:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417245:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.417249:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417258:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.417260:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204893.417265:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417270:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204893.417273:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301024769 00000100:00000010:3.0:1788204893.417279:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f47e0. 00000100:00000001:3.0:1788204893.417285:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204893.417298:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301024769 00000100:00000010:3.0:1788204893.417301:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4cc0. 00000100:00000001:3.0:1788204893.417305:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204893.417308:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204893.417313:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.417316:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204893.417321:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204893/real 1788204893] req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204893.417331:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204893.417333:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204893.417335:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204893.417338:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204893.417340:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204893.417343:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204893.417347:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204893.417356:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417399:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204893.417402:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417405:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.417407:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204893.417412:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417420:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204893.417424:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417439:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204893.417443:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417445:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.417446:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417449:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204893.417451:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204893.417455:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204893.417459:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204893.417462:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204893.417466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204893.417476:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205908096:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204893.417481:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417484:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.417487:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205908096/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417494:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204893.417496:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204893.417499:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204893.417502:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204893.417504:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204893.417506:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204893.417509:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204893.417511:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788204893.417515:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204893.417524:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204893.417526:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204893.417530:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204893.417531:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417533:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417541:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204893.417543:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.417545:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204893.417547:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204893.417548:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204893.417552:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417560:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204893.417562:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.417564:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204893.417566:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204893.417568:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204893.417572:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908224:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204893.417575:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204893.417577:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204893.417579:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.417581:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204893.417585:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204893.417587:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417591:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204893.417594:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204893.417598:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204893.417602:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908224, portal 10 00000100:00000001:3.0:1788204893.417604:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204893.417605:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204893.417612:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417620:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204893.417622:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204893.417625:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204893.417627:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205908224, offset 0 00000400:00000200:3.0:1788204893.417641:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204893.417645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.417650:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204893.417653:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204893.417656:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204893.417659:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204893.417661:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204893.417665:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58951:58951:256:4294967295] 192.168.201.151@tcp LPNI seq info [58931:58931:8:4294967295] 00000400:00000200:3.0:1788204893.417669:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.417677:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204893.417682:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.417686:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204893.417691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204893.417694:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204893.417699:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204893.417702:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204893.417706:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.417708:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204893.417711:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204893.417715:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204893.417719:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204893.417721:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204893.417724:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204893.417728:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417737:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.417740:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417747:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.417749:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204893.417752:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204893.417756:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204893.417760:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204893.417762:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301020827 00000100:00000001:3.0:1788204893.417766:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.417768:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301020827 00000100:00000001:3.0:1788204893.417770:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.417773:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204893.417775:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.417777:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204893.417781:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204893/real 1788204893] req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204893.417790:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204893.417791:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204893.417793:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204893.417796:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204893.417798:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204893.417800:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204893.417803:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204893.417813:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417820:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204893.417822:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417824:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.417825:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204893.417829:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417837:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204893.417841:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417849:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204893.417851:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417853:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.417855:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.417857:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204893.417859:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204893.417862:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204893.417865:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204893.417868:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204893.417872:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204893.417881:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908224:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204893.417885:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204893.417889:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417896:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204893.417898:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.417900:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204893.417902:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204893.417904:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204893.417907:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204893.417910:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204893.417912:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204893.417914:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.417916:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204893.417919:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204893.417922:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.417925:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204893.417928:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204893.417932:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204893.417934:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908352, portal 4 00000100:00000001:3.0:1788204893.417937:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204893.417938:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204893.417945:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.417952:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204893.417954:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204893.417957:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204893.417959:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205908352, offset 0 00000400:00000200:3.0:1788204893.417964:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204893.417968:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.417972:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204893.417976:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204893.417978:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204893.417981:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204893.417984:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204893.417987:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58952:58952:256:4294967295] 192.168.201.151@tcp LPNI seq info [58932:58932:8:4294967295] 00000400:00000200:3.0:1788204893.417992:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.417998:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204893.418003:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.418007:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204893.418011:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204893.418014:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204893.418031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204893.418038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204893.418043:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.418045:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204893.418048:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204893.418052:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204893.418055:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204893.418057:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204893.418060:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204893.418064:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418072:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.418075:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418083:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.418084:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204893.418088:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204893.418091:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204893.418094:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204893.418096:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301020713 00000100:00000001:3.0:1788204893.418099:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.418101:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301020713 00000100:00000001:3.0:1788204893.418104:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.418106:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204893.418109:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.418111:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204893.418115:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204893/real 1788204893] req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204893.418124:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204893.418125:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204893.418127:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204893.418130:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204893.418132:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204893.418134:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204893.418138:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204893.418147:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418155:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204893.418157:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418158:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.418160:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204893.418164:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418172:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204893.418176:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418184:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204893.418186:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418187:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.418188:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418190:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204893.418192:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204893.418195:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204893.418198:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204893.418200:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204893.418205:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204893.418214:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204893.418218:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204893.418221:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418229:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204893.418230:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.418233:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204893.418235:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204893.418237:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204893.418240:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908480:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204893.418243:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204893.418245:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204893.418247:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204893.418249:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204893.418252:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204893.418254:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204893.418258:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204893.418260:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204893.418263:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204893.418266:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908480, portal 4 00000100:00000001:3.0:1788204893.418268:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204893.418270:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204893.418276:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418282:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204893.418285:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204893.418287:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204893.418289:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205908480, offset 0 00000400:00000200:3.0:1788204893.418294:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204893.418297:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.418301:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204893.418304:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204893.418306:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204893.418309:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204893.418312:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204893.418315:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58953:58953:256:4294967295] 192.168.201.151@tcp LPNI seq info [58933:58933:8:4294967295] 00000400:00000200:3.0:1788204893.418320:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204893.418326:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204893.418331:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.418335:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204893.418339:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204893.418342:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204893.418347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204893.418350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204893.418354:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204893.418356:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204893.418360:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204893.418395:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204893.418399:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204893.418401:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204893.418404:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204893.418408:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418416:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.418420:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204918 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418427:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.418428:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204893.418432:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204893.418435:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204893.418438:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204893.418441:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301020720 00000100:00000001:3.0:1788204893.418443:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.418445:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301020720 00000100:00000001:3.0:1788204893.418448:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301024769 00000100:00000001:3.0:1788204893.418450:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204893.418452:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.418454:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204893.418458:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204893/real 1788204893] req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204893.418467:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204893.418468:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204893.418470:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204893.418472:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204893.418474:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204893.418476:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204893.418480:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204893.418489:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418495:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204893.418497:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418499:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.418500:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204893.418504:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418511:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204893.418515:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418523:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204893.418526:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418528:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204893.418529:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418531:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204893.418533:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204893.418536:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204893.418540:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204893.418542:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204893.418546:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204893.418554:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908480:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204893.418558:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418559:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.418563:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205908224/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418571:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204893.418573:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204893.418575:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204893.418577:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204893.418579:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204893.418580:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204893.418583:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204893.418585:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788204893.418587:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204893.418589:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204893.418590:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204893.418593:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204893.418595:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418596:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.418600:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205908352/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418608:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204893.418609:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204893.418611:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204893.418613:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204893.418615:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204893.418616:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204893.418618:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204893.418620:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788204893.418622:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204893.418623:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204893.418624:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204893.418627:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204893.418628:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418630:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204893.418634:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205908480/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204918 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204893.418641:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204893.418643:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204893.418644:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204893.418647:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204893.418648:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204893.418649:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204893.418652:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204893.418653:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788204893.418655:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204893.418657:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204893.418658:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204893.418660:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204893.418662:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418664:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.418667:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204893.418669:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.418671:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204893.418672:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204893.608527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204893.608537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204893.608542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204893.608543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204893.608545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204893.608546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204893.608548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204893.608549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204893.608563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204893.608566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204893.608568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204893.928496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204893.928504:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204893.928505:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204893.928510:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204893.928513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204893.928515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.928518:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204893.928520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204893.928523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204894.376486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204894.376492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204894.376494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204894.376499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204894.376502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204894.376505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204894.376507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204894.376509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204894.376511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204894.440514:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204894.440519:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204894.440521:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204894.440527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204894.440531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204894.440535:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204894.440538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204894.440541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204894.440543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204894.440582:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204894.440587:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204894.440589:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204894.440592:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204894.440595:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204894.440597:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204894.440599:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204894.440601:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204894.440603:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204894.632494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204894.632500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204894.632505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204894.632507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204894.632509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204894.632510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204894.632513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204894.632515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204894.632592:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204894.632596:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204894.632599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204894.952494:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204894.952502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204894.952504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204894.952509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204894.952512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204894.952515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204894.952517:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204894.952520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204894.952522:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204895.400446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204895.400453:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204895.400455:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204895.400459:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204895.400463:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204895.400467:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204895.400469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204895.400472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204895.400475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204895.464485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204895.464494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204895.464496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204895.464503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204895.464507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204895.464511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204895.464514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204895.464517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204895.464521:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.464522:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204895.464527:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204895.464529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.464534:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204895.464537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204895.464540:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.464542:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.464544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.464547:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204895.656530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204895.656538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204895.656544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204895.656545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204895.656548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204895.656550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204895.656553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204895.656554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204895.656576:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204895.656578:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204895.656581:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204895.976489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204895.976495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204895.976496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.976501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204895.976504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204895.976507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.976509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.976511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.976513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204896.424489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204896.424497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204896.424499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204896.424505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204896.424509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204896.424512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204896.424515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204896.424518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204896.424521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204896.488450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204896.488450:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204896.488457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204896.488458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204896.488460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204896.488461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204896.488468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204896.488469:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204896.488474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204896.488475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204896.488480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204896.488480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204896.488486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204896.488489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204896.488491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204896.488492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204896.488495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204896.488498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204896.680443:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204896.680449:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204896.680454:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204896.680455:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204896.680457:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204896.680458:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204896.680461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204896.680462:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204896.680516:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204896.680520:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204896.680522:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204897.000456:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204897.000461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204897.000463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204897.000467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204897.000469:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204897.000472:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204897.000475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204897.000478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204897.000480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204897.448424:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204897.448431:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204897.448432:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204897.448436:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204897.448439:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204897.448442:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204897.448444:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204897.448446:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204897.448448:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204897.512473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204897.512473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204897.512480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204897.512481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204897.512484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204897.512485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204897.512490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204897.512491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204897.512496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204897.512496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204897.512501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204897.512501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204897.512507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204897.512510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204897.512512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204897.512513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204897.512516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204897.512518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204897.704509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204897.704514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204897.704518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204897.704519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204897.704522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204897.704522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204897.704525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204897.704526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204897.704540:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204897.704542:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204897.704544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204898.024486:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301024769, at 4301025280 00000100:00000040:1.0:1788204898.024486:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301024769, at 4301025280 00000100:00000001:3.0:1788204898.024491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204898.024497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204898.024502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204898.024503:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000010:1.0:1788204898.024505:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00100000:3.0:1788204898.024506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204898.024513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204898.024516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.024519:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204898.024522:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.024526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.472481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204898.472488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204898.472490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204898.472494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204898.472496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204898.472500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204898.472501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204898.472503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204898.472505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204898.536448:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204898.536448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204898.536455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204898.536456:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204898.536458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.536460:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204898.536464:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204898.536466:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204898.536468:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204898.536473:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.536473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788204898.536475:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204898.536476:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204898.536478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204898.536480:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204898.536483:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204898.536488:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536488:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204898.536488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788204898.536493:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204898.536495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204898.536498:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204898.536500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204898.536505:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6353 00000400:00000200:1.0:1788204898.536514:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204898.536520:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204898.536523:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204898.536525:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536527:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.536529:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536531:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204898.536534:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204898.536535:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204898.536537:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204898.536540:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204898.536541:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204898.536543:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204898.536545:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536547:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.536548:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536549:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204898.536552:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204898.536554:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536556:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204898.536557:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204898.536560:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204898.536562:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536563:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204898.536565:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204898.536573:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204898.536576:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204898.536579:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204898.536581:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536583:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204898.536585:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204898.536588:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204898.536594:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788204898.536599:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204898.536606:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204898.536614:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204898.536618:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204898.536621:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204898.536638:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204898.536641:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204898.536643:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204898.536644:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204898.536645:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204898.536647:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204898.536649:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6353 00000400:00000200:1.0:1788204898.536651:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204898.536653:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204898.536655:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204898.536657:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536659:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.536661:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536662:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204898.536664:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204898.536666:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204898.536667:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204898.536669:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204898.536669:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788204898.536674:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204898.536675:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788204898.536676:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204898.536678:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204898.536680:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536682:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.536686:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204898.536687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204898.536688:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204898.536696:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204898.536700:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536704:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.536704:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788204898.536708:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204898.536709:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204898.536712:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204898.536716:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.536716:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788204898.536720:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204898.536721:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1788204898.536723:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204898.536727:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.536728:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536729:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204898.536735:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788204898.536740:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00100000:3.0:1788204898.536740:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205908608:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788204898.536744:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204898.536745:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536748:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204898.536750:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204898.536751:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204898.536754:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000100:00000001:1.0:1788204898.536756:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.536759:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788204898.536762:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204898.536767:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788204898.536768:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204898.536771:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.536771:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.536777:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1788204898.536778:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788204898.536784:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204898.536788:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536796:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.536797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788204898.536799:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204898.536801:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00080000:1.0:1788204898.536803:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204898.536806:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204898.536807:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788204898.536809:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:1.0:1788204898.536810:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204898.536814:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6353 00000400:00000200:1.0:1788204898.536817:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:3.0:1788204898.536817:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908608, portal 25 00000100:00000001:3.0:1788204898.536822:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1788204898.536823:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204898.536825:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00080000:1.0:1788204898.536826:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204898.536831:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536834:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204898.536835:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204898.536837:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536841:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204898.536844:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204898.536846:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204898.536848:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204898.536850:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204898.536853:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204898.536855:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204898.536859:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204898.536859:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204898.536861:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.536862:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536864:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204898.536865:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204898.536867:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536870:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204898.536872:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1788204898.536876:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788204898.536877:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204898.536880:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536881:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204898.536883:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204898.536884:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536887:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204898.536890:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000400:00000010:3.0:1788204898.536890:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000020:00000040:1.0:1788204898.536895:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000200:3.0:1788204898.536896:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205908608, offset 0 02000000:00000001:1.0:1788204898.536897:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204898.536899:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.536902:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204898.536903:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204898.536904:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204898.536909:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788204898.536911:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.536912:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00100000:1.0:1788204898.536916:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204898.536922:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204898.536924:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204898.536925:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204898.536930:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204898.536936:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.536937:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:1.0:1788204898.536938:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204898.536944:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204898.536946:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204898.536947:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204898.536949:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204898.536949:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1788204898.536954:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6353 00000400:00000200:1.0:1788204898.536958:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.536959:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788204898.536962:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204898.536964:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204898.536968:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1788204898.536969:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536972:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204898.536975:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:1.0:1788204898.536977:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.536979:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204898.536983:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788204898.536983:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58954:58954:256:4294967295] 192.168.201.151@tcp LPNI seq info [58934:58934:8:4294967295] 00000020:00000040:1.0:1788204898.536986:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204898.536988:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204898.536991:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:3.0:1788204898.536993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788204898.536994:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204898.536998:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204898.537001:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.537002:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204898.537007:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204898.537011:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204898.537011:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:1.0:1788204898.537014:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204898.537019:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204898.537023:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204898.537024:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788204898.537025:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204898.537028:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204898.537030:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204898.537031:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204898.537032:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204898.537034:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204898.537034:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:1.0:1788204898.537037:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204898.537042:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204898.537044:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204898.537048:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204898.537049:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204898.537049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788204898.537053:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204898.537055:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204898.537056:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204898.537058:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 02000000:00000010:1.0:1788204898.537059:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788204898.537064:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204898.537069:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204898.537070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204898.537077:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1788204898.537080:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204898.537083:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204898.537086:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788204898.537090:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1788204898.537091:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204898.537099:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788204898.537100:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204898.537106:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6363) 00000400:00000200:3.0:1788204898.537110:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204898.537115:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204898.537117:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204898.537121:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204898.537125:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537133:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537135:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537141:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537142:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204898.537146:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.537149:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204898.537152:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301029889 00000100:00000010:3.0:1788204898.537156:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f43c0. 00000100:00000001:3.0:1788204898.537162:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204898.537166:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301029889 00000100:00000010:3.0:1788204898.537168:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4f60. 00000100:00000001:3.0:1788204898.537171:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204898.537173:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204898.537175:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537177:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204898.537180:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204898/real 1788204898] req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204898.537186:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204898.537187:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204898.537189:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204898.537190:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204898.537192:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204898.537193:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204898.537196:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204898.537209:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537214:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204898.537215:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537217:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537219:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204898.537221:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537226:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204898.537229:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537234:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204898.537235:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537236:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537237:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537239:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204898.537240:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204898.537242:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204898.537245:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204898.537247:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204898.537249:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204898.537255:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205908608:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204898.537258:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537259:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537261:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205908608/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537269:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204898.537271:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204898.537273:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204898.537275:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204898.537277:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204898.537278:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204898.537290:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204898.537292:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:3.0:1788204898.537295:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204898.537297:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204898.537298:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:3.0:1788204898.537300:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204898.537301:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537303:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537308:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204898.537309:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537311:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204898.537312:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204898.537313:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204898.537316:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537321:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204898.537322:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537323:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204898.537325:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204898.537326:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204898.537328:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908736:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204898.537330:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204898.537332:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204898.537333:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537334:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204898.537336:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204898.537338:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.537340:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204898.537343:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204898.537345:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204898.537347:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908736, portal 10 00000100:00000001:3.0:1788204898.537348:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204898.537349:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:3.0:1788204898.537354:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537359:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204898.537379:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204898.537382:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204898.537383:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205908736, offset 0 00000400:00000200:3.0:1788204898.537386:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537389:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537392:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204898.537394:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204898.537396:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204898.537398:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204898.537400:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537402:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58955:58955:256:4294967295] 192.168.201.151@tcp LPNI seq info [58935:58935:8:4294967295] 00000400:00000200:3.0:1788204898.537404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537409:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204898.537412:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537414:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204898.537417:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204898.537419:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204898.537423:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204898.537435:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204898.537438:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537440:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204898.537442:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204898.537444:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204898.537447:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204898.537448:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204898.537451:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204898.537453:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537459:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537461:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537465:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537466:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204898.537469:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204898.537471:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204898.537473:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204898.537475:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301025947 00000100:00000001:3.0:1788204898.537477:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537478:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301025947 00000100:00000001:3.0:1788204898.537480:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537481:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204898.537483:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537484:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204898.537486:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204898/real 1788204898] req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204898.537491:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204898.537492:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204898.537493:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204898.537494:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204898.537495:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204898.537497:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204898.537499:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204898.537504:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537512:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204898.537513:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537514:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537515:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204898.537517:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537521:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204898.537524:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537528:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204898.537529:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537530:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537531:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537532:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204898.537534:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204898.537535:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204898.537537:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204898.537538:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204898.537540:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204898.537545:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908736:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204898.537547:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204898.537549:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537554:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204898.537555:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537556:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204898.537558:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204898.537559:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204898.537561:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908864:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204898.537563:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204898.537564:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204898.537565:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537566:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204898.537568:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204898.537569:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.537572:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204898.537573:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204898.537576:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204898.537577:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908864, portal 4 00000100:00000001:3.0:1788204898.537579:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204898.537580:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:3.0:1788204898.537584:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537588:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204898.537590:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204898.537591:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204898.537592:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205908864, offset 0 00000400:00000200:3.0:1788204898.537595:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537597:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537599:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204898.537601:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204898.537602:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204898.537604:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204898.537606:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537608:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58956:58956:256:4294967295] 192.168.201.151@tcp LPNI seq info [58936:58936:8:4294967295] 00000400:00000200:3.0:1788204898.537610:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537615:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204898.537618:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537620:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204898.537622:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204898.537624:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204898.537627:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204898.537629:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204898.537632:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537633:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204898.537635:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204898.537637:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204898.537639:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204898.537640:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204898.537641:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204898.537644:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537648:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537650:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537654:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537655:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204898.537657:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204898.537659:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204898.537661:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204898.537662:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301025833 00000100:00000001:3.0:1788204898.537664:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537665:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301025833 00000100:00000001:3.0:1788204898.537667:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537668:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204898.537669:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537670:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204898.537672:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204898/real 1788204898] req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204898.537678:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204898.537679:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204898.537680:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204898.537681:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204898.537682:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204898.537683:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204898.537685:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204898.537690:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537695:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204898.537696:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537698:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537699:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204898.537703:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537710:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204898.537713:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537720:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204898.537722:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537724:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537726:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537728:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204898.537730:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204898.537733:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204898.537737:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204898.537739:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204898.537744:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204898.537752:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908864:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204898.537754:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204898.537756:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537761:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204898.537762:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537763:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204898.537765:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204898.537766:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204898.537768:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908992:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204898.537770:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204898.537771:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204898.537772:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204898.537773:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204898.537776:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204898.537777:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204898.537780:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204898.537781:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204898.537783:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204898.537785:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205908992, portal 4 00000100:00000001:3.0:1788204898.537787:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204898.537787:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:3.0:1788204898.537791:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537796:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204898.537797:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204898.537799:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204898.537800:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205908992, offset 0 00000400:00000200:3.0:1788204898.537803:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537806:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537808:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204898.537810:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204898.537812:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204898.537814:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204898.537816:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204898.537818:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58957:58957:256:4294967295] 192.168.201.151@tcp LPNI seq info [58937:58937:8:4294967295] 00000400:00000200:3.0:1788204898.537820:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204898.537825:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204898.537827:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537829:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204898.537832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204898.537834:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204898.537836:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204898.537839:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204898.537842:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204898.537844:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204898.537847:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204898.537851:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204898.537853:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204898.537855:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204898.537858:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204898.537861:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537869:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537873:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537881:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537882:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204898.537885:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204898.537888:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204898.537892:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204898.537893:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301025839 00000100:00000001:3.0:1788204898.537895:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537897:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301025839 00000100:00000001:3.0:1788204898.537898:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301029889 00000100:00000001:3.0:1788204898.537900:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204898.537901:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.537903:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204898.537905:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204898/real 1788204898] req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204898.537911:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204898.537912:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204898.537913:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204898.537914:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204898.537916:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204898.537917:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204898.537920:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204898.537930:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537937:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204898.537938:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537940:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537942:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204898.537946:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537953:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204898.537958:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537966:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204898.537968:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537969:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204898.537970:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537971:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204898.537972:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204898.537974:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204898.537977:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204898.537978:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204898.537981:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204898.537986:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205908992:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204898.537988:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.537990:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.537992:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205908736/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.537996:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204898.537997:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204898.537998:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204898.537999:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204898.538001:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204898.538002:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204898.538003:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204898.538005:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:3.0:1788204898.538006:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204898.538007:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204898.538008:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:3.0:1788204898.538010:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204898.538011:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.538012:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.538014:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205908864/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.538025:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204898.538026:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204898.538028:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204898.538031:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204898.538032:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204898.538033:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204898.538035:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204898.538037:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:3.0:1788204898.538039:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204898.538041:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204898.538042:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:3.0:1788204898.538044:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204898.538046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.538048:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204898.538052:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205908992/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204898.538059:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204898.538060:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204898.538062:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204898.538064:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204898.538065:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204898.538067:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204898.538068:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204898.538069:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:3.0:1788204898.538071:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204898.538072:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204898.538073:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:3.0:1788204898.538074:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204898.538075:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.538076:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204898.538079:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204898.538080:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.538081:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204898.538083:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204898.728478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204898.728483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204898.728487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204898.728489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204898.728491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204898.728492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204898.728495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204898.728495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204898.728557:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204898.728560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204898.728562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204899.048438:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204899.048446:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204899.048448:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204899.048453:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204899.048457:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204899.048461:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204899.048464:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204899.048467:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204899.048470:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204899.496482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204899.496489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204899.496491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204899.496496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204899.496500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204899.496504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204899.496506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204899.496509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204899.496512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204899.560444:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204899.560451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204899.560452:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204899.560453:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204899.560458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788204899.560459:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204899.560463:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204899.560464:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204899.560469:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204899.560470:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204899.560476:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204899.560476:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204899.560480:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204899.560481:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204899.560486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204899.560488:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204899.560492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204899.560495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204899.752491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204899.752496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204899.752500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204899.752501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204899.752504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204899.752505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204899.752508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204899.752509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204899.752524:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204899.752527:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204899.752530:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204900.072475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204900.072484:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204900.072486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204900.072492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204900.072496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204900.072505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.072508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.072511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.072515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204900.520474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204900.520480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204900.520482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204900.520486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204900.520489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204900.520492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204900.520494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204900.520495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204900.520497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204900.584483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204900.584483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204900.584490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204900.584490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204900.584492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.584492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204900.584497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204900.584497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204900.584501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204900.584501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204900.584505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.584505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204900.584508:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204900.584511:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204900.584514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.584518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.584520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.584522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204900.776494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204900.776505:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204900.776507:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204900.776510:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776513:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204900.776516:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776518:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776521:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204900.776525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204900.776529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204900.776530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204900.776595:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204900.776599:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204900.776602:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204901.096423:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204901.096429:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204901.096431:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204901.096435:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204901.096439:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204901.096442:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.096445:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204901.096447:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.096449:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204901.544466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204901.544475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204901.544477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204901.544483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204901.544487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204901.544491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204901.544493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204901.544496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204901.544499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.608478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204901.608480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204901.608488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204901.608488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204901.608492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.608493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204901.608500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204901.608500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204901.608507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204901.608507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204901.608513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.608514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204901.608521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204901.608526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204901.608530:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.608531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204901.608535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.608538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204901.800490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204901.800496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204901.800500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204901.800501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204901.800503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204901.800504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204901.800506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204901.800507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204901.800521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204901.800523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204901.800525:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204902.120434:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204902.120440:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204902.120441:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204902.120445:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204902.120448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204902.120451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.120453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204902.120456:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.120459:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204902.568479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204902.568485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204902.568487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204902.568491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204902.568494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204902.568497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204902.568499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204902.568501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204902.568503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204902.632467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204902.632467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204902.632473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204902.632474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204902.632476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204902.632479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204902.632482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204902.632485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204902.632486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204902.632492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.632492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204902.632495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204902.632497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204902.632498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204902.632504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204902.632504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204902.632508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.632512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204902.824505:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204902.824507:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204902.824510:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824515:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204902.824517:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824519:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824520:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204902.824526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204902.824529:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204902.824530:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204902.824531:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824532:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204902.824533:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824534:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824535:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204902.824539:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204902.824540:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204902.824541:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824542:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204902.824543:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824544:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204902.824546:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204902.824621:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204902.824626:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204902.824629:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204903.144424:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204903.144433:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204903.144437:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204903.144442:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301029889, at 4301030400 00000100:00000040:1.0:1788204903.144442:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301029889, at 4301030400 00000100:00100000:3.0:1788204903.144444:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204903.144449:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204903.144455:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204903.144456:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000010:1.0:1788204903.144459:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000001:3.0:1788204903.144459:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.144464:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.144469:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.592478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204903.592484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204903.592486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204903.592490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204903.592493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204903.592496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204903.592498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204903.592500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204903.592502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204903.656451:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204903.656451:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204903.656456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204903.656457:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204903.656461:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204903.656461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204903.656462:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204903.656465:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788204903.656468:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204903.656468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204903.656470:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204903.656475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204903.656475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204903.656476:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204903.656480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204903.656481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788204903.656484:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6358 00000100:00000001:0.0:1788204903.656487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.656487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.656491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204903.656493:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204903.656496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204903.656500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1788204903.656501:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788204903.656506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204903.656506:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788204903.656513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204903.656513:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656525:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.656528:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656530:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204903.656533:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204903.656535:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204903.656538:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204903.656540:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204903.656542:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204903.656544:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204903.656546:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656548:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.656549:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656550:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204903.656552:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204903.656555:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656556:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204903.656558:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204903.656561:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204903.656564:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656565:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204903.656566:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204903.656573:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788204903.656576:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204903.656579:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204903.656580:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656583:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204903.656584:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204903.656588:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204903.656594:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788204903.656599:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204903.656606:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204903.656613:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204903.656616:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204903.656620:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204903.656636:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204903.656638:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204903.656640:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204903.656642:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204903.656643:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204903.656645:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204903.656647:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6358 00000400:00000200:1.0:1788204903.656650:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204903.656653:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204903.656654:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204903.656656:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204903.656658:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204903.656659:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204903.656663:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788204903.656664:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204903.656667:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788204903.656668:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204903.656671:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204903.656672:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204903.656675:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:3.0:1788204903.656676:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204903.656677:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204903.656681:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204903.656685:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204903.656691:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656695:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204903.656695:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.656698:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204903.656700:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656703:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788204903.656703:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1788204903.656705:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204903.656706:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788204903.656710:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204903.656711:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656713:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204903.656715:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788204903.656717:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205909120:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788204903.656720:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204903.656721:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204903.656725:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.656726:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788204903.656727:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204903.656729:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788204903.656732:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204903.656732:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1788204903.656736:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656738:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204903.656739:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000010:1.0:1788204903.656741:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 02000000:00000001:3.0:1788204903.656743:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204903.656746:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204903.656749:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204903.656751:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656753:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788204903.656753:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788204903.656756:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204903.656759:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204903.656762:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 02000000:00000010:1.0:1788204903.656763:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788204903.656768:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204903.656773:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204903.656774:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:1.0:1788204903.656782:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204903.656784:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909120, portal 25 00000100:00000001:1.0:1788204903.656785:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.656792:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204903.656795:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:1.0:1788204903.656796:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204903.656805:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204903.656807:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204903.656812:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204903.656819:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204903.656822:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204903.656825:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204903.656826:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204903.656828:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788204903.656832:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788204903.656835:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6358 00000400:00000010:3.0:1788204903.656837:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000400:00000200:1.0:1788204903.656841:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:3.0:1788204903.656844:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205909120, offset 0 00000400:00000200:1.0:1788204903.656845:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204903.656849:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204903.656853:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.656857:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656858:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.656862:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656864:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204903.656866:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204903.656868:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788204903.656869:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788204903.656871:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204903.656874:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788204903.656875:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204903.656878:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:1.0:1788204903.656879:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204903.656885:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.656886:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204903.656888:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.656891:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.656893:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788204903.656894:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204903.656897:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204903.656899:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1788204903.656903:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204903.656906:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.656907:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204903.656910:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204903.656914:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204903.656914:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58958:58958:256:4294967295] 192.168.201.151@tcp LPNI seq info [58938:58938:8:4294967295] 02000000:00000001:1.0:1788204903.656919:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656922:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204903.656923:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:1.0:1788204903.656925:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788204903.656927:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656928:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204903.656930:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788204903.656932:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204903.656933:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204903.656935:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.656938:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204903.656939:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204903.656940:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204903.656943:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000400:00000200:3.0:1788204903.656947:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788204903.656949:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204903.656956:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788204903.656957:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.656965:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:1.0:1788204903.656966:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204903.656969:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204903.656973:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788204903.656976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204903.656982:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1788204903.656984:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204903.656988:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204903.656993:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:1.0:1788204903.656994:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204903.656997:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788204903.656998:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00080000:1.0:1788204903.656999:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204903.657004:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:3.0:1788204903.657006:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788204903.657009:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6358 00000800:00000010:3.0:1788204903.657010:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:1.0:1788204903.657014:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657017:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204903.657018:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204903.657022:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204903.657026:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1788204903.657027:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.657030:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.657032:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657032:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000100:00000001:1.0:1788204903.657037:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000010:3.0:1788204903.657037:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000020:00000001:1.0:1788204903.657039:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204903.657041:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204903.657042:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204903.657043:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1788204903.657044:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204903.657047:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204903.657051:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000200:3.0:1788204903.657051:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204903.657057:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.657061:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204903.657064:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204903.657068:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204903.657071:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204903.657074:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788204903.657075:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204903.657079:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204903.657081:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204903.657084:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788204903.657084:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204903.657085:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.657088:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204903.657090:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788204903.657094:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.657098:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204903.657102:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788204903.657107:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204903.657108:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204903.657109:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204903.657114:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.657115:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204903.657116:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204903.657117:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204903.657119:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788204903.657121:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657123:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1788204903.657125:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204903.657131:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204903.657131:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204903.657135:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204903.657139:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:1.0:1788204903.657140:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657143:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301035009 00000100:00000001:1.0:1788204903.657151:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204903.657151:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4d20. 00000100:00000040:1.0:1788204903.657154:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6368) 00000100:00000001:3.0:1788204903.657158:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204903.657165:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301035009 00000100:00000010:3.0:1788204903.657168:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4480. 00000100:00000001:3.0:1788204903.657171:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204903.657174:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204903.657176:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657177:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204903.657182:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204903/real 1788204903] req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204903.657188:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204903.657189:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204903.657203:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204903.657206:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204903.657207:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204903.657209:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204903.657212:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204903.657218:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657223:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204903.657224:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657226:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657227:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204903.657230:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657235:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204903.657238:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657243:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204903.657245:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657246:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657246:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657248:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204903.657249:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204903.657251:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204903.657254:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204903.657255:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204903.657258:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204903.657263:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205909120:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204903.657266:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657267:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657269:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205909120/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657273:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204903.657275:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657277:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204903.657279:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204903.657280:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204903.657281:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204903.657283:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204903.657285:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:3.0:1788204903.657287:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204903.657289:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204903.657290:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:3.0:1788204903.657292:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204903.657293:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657295:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657300:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204903.657301:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657303:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204903.657304:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204903.657305:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204903.657307:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657312:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.657313:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657315:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204903.657316:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204903.657317:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204903.657319:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909248:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204903.657321:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204903.657322:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204903.657323:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657325:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657327:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204903.657328:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204903.657331:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204903.657333:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204903.657335:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204903.657337:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909248, portal 10 00000100:00000001:3.0:1788204903.657338:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204903.657339:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:3.0:1788204903.657344:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657348:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204903.657350:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204903.657351:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204903.657353:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205909248, offset 0 00000400:00000200:3.0:1788204903.657356:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657359:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657361:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204903.657381:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204903.657383:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204903.657385:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204903.657387:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657389:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58959:58959:256:4294967295] 192.168.201.151@tcp LPNI seq info [58939:58939:8:4294967295] 00000400:00000200:3.0:1788204903.657392:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657397:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204903.657400:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657403:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204903.657405:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204903.657407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204903.657411:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204903.657412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204903.657415:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657416:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204903.657418:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204903.657421:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204903.657423:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204903.657424:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204903.657426:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204903.657429:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657434:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657436:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657441:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657442:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204903.657444:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204903.657447:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204903.657449:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204903.657451:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301031067 00000100:00000001:3.0:1788204903.657453:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657454:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301031067 00000100:00000001:3.0:1788204903.657456:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657457:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204903.657458:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657459:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204903.657462:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204903/real 1788204903] req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204903.657466:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204903.657467:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204903.657469:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204903.657470:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204903.657471:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204903.657472:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204903.657474:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204903.657480:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657484:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204903.657485:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657486:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657487:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204903.657489:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657493:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204903.657495:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657500:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204903.657501:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657503:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657503:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657504:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204903.657506:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204903.657507:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204903.657509:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204903.657511:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204903.657513:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204903.657518:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909248:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204903.657520:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204903.657522:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657526:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.657527:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657529:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204903.657530:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204903.657531:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204903.657533:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909376:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204903.657535:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204903.657536:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204903.657537:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657538:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657540:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204903.657541:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204903.657543:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204903.657545:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204903.657547:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204903.657549:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909376, portal 4 00000100:00000001:3.0:1788204903.657550:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204903.657551:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:3.0:1788204903.657555:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657559:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204903.657561:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204903.657562:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204903.657564:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205909376, offset 0 00000400:00000200:3.0:1788204903.657566:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657568:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657571:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204903.657573:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204903.657575:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204903.657576:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204903.657578:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657580:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58960:58960:256:4294967295] 192.168.201.151@tcp LPNI seq info [58940:58940:8:4294967295] 00000400:00000200:3.0:1788204903.657582:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657586:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204903.657589:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657591:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204903.657593:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204903.657595:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204903.657598:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204903.657600:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204903.657602:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657603:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204903.657605:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204903.657608:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204903.657609:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204903.657610:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204903.657612:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204903.657615:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657619:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657621:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657626:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657627:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204903.657629:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204903.657631:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204903.657632:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204903.657634:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301030953 00000100:00000001:3.0:1788204903.657635:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657637:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301030953 00000100:00000001:3.0:1788204903.657638:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657639:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204903.657641:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657642:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204903.657644:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204903/real 1788204903] req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204903.657648:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204903.657649:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204903.657650:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204903.657651:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204903.657652:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204903.657653:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204903.657655:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204903.657660:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657668:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204903.657669:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657670:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657671:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204903.657673:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657677:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204903.657679:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657683:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204903.657685:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657686:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657686:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657688:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204903.657689:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204903.657691:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204903.657692:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204903.657694:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204903.657696:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204903.657701:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909376:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204903.657703:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204903.657705:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657709:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.657710:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657712:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204903.657713:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204903.657714:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204903.657716:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909504:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204903.657718:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204903.657719:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204903.657720:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204903.657721:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657723:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204903.657724:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204903.657726:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204903.657728:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204903.657729:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204903.657731:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909504, portal 4 00000100:00000001:3.0:1788204903.657732:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204903.657733:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00000040:3.0:1788204903.657737:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657740:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204903.657742:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204903.657743:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204903.657745:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205909504, offset 0 00000400:00000200:3.0:1788204903.657747:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657750:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657753:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204903.657756:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204903.657759:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204903.657762:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204903.657764:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204903.657767:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58961:58961:256:4294967295] 192.168.201.151@tcp LPNI seq info [58941:58941:8:4294967295] 00000400:00000200:3.0:1788204903.657772:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204903.657778:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204903.657783:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657787:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204903.657791:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204903.657793:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204903.657799:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204903.657801:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204903.657806:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204903.657808:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204903.657810:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204903.657813:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204903.657815:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:3.0:1788204903.657816:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:3.0:1788204903.657818:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204903.657821:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657826:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657828:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657832:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657833:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204903.657836:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204903.657838:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204903.657840:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204903.657841:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301030959 00000100:00000001:3.0:1788204903.657843:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657845:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301030959 00000100:00000001:3.0:1788204903.657846:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301035009 00000100:00000001:3.0:1788204903.657847:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204903.657849:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657850:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204903.657852:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204903/real 1788204903] req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204903.657858:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204903.657859:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204903.657860:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204903.657861:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204903.657862:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204903.657864:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204903.657866:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204903.657871:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657875:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204903.657876:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657877:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657878:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204903.657880:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657884:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204903.657886:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657890:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204903.657891:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657892:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204903.657893:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657895:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204903.657896:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204903.657898:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204903.657900:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204903.657901:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204903.657904:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204903.657909:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909504:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204903.657912:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657913:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657915:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205909248/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657918:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204903.657920:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657921:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204903.657922:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204903.657923:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204903.657924:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204903.657925:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204903.657926:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:3.0:1788204903.657928:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204903.657929:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204903.657930:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:3.0:1788204903.657931:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204903.657932:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657933:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657935:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205909376/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657939:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204903.657940:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657941:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204903.657942:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204903.657943:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204903.657943:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204903.657945:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204903.657945:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:3.0:1788204903.657947:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204903.657948:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204903.657949:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:3.0:1788204903.657950:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204903.657951:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657952:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204903.657954:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205909504/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204903.657958:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204903.657958:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204903.657959:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204903.657960:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204903.657961:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204903.657962:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204903.657963:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204903.657964:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:3.0:1788204903.657965:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204903.657965:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204903.657966:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:3.0:1788204903.657968:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204903.657968:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657970:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204903.657972:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204903.657973:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.657974:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204903.657975:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204903.848515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204903.848521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204903.848526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204903.848527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204903.848529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204903.848530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204903.848532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204903.848533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204903.848548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204903.848550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204903.848553:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204904.168410:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204904.168416:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204904.168418:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204904.168422:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204904.168424:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204904.168427:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.168429:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204904.168431:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.168433:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204904.616516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204904.616525:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204904.616527:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204904.616534:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204904.616538:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204904.616542:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204904.616545:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204904.616547:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204904.616552:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204904.680424:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204904.680435:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204904.680438:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204904.680445:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204904.680451:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204904.680457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.680461:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204904.680464:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.680469:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204904.680478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204904.680486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204904.680489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204904.680495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204904.680498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204904.680503:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204904.680507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204904.680510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204904.680513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204904.872513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204904.872519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204904.872525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204904.872527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204904.872530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204904.872532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204904.872535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204904.872536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204904.872607:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204904.872611:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204904.872615:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204905.192420:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204905.192430:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204905.192433:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.192441:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204905.192446:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.192450:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.192453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.192456:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.192458:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204905.640498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204905.640506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204905.640508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204905.640514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204905.640518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204905.640522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204905.640526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204905.640528:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204905.640532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204905.704471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204905.704471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204905.704479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204905.704480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204905.704484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.704484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204905.704493:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204905.704493:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204905.704501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.704501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.704507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204905.704508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.704512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.704515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.704518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204905.704522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204905.704526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204905.704529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204905.896563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204905.896570:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204905.896576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204905.896578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204905.896581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204905.896583:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204905.896586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204905.896588:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204905.896609:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204905.896613:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204905.896616:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204906.216411:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204906.216418:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204906.216420:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204906.216425:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204906.216430:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204906.216433:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204906.216436:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204906.216439:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204906.216442:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204906.664501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204906.664509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204906.664512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204906.664517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204906.664522:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204906.664526:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204906.664529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204906.664531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204906.664534:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204906.728487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204906.728487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204906.728494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204906.728495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204906.728497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204906.728499:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204906.728503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204906.728505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204906.728507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204906.728512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204906.728513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204906.728517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204906.728518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204906.728521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204906.728526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204906.728526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204906.728531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204906.728536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204906.920456:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204906.920462:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204906.920468:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204906.920469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204906.920472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204906.920474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204906.920477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204906.920478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204906.920560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204906.920564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204906.920567:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204907.240422:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204907.240429:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204907.240431:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204907.240435:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204907.240438:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204907.240441:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.240443:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204907.240445:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.240448:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204907.688483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204907.688490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204907.688492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204907.688508:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204907.688512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204907.688516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204907.688518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204907.688521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204907.688525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204907.752485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204907.752494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204907.752496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204907.752499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204907.752503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204907.752504:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204907.752506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204907.752508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204907.752511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204907.752514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204907.752515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204907.752516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.752519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204907.752523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.752525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204907.752527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204907.752530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204907.752533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204907.944597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204907.944605:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204907.944612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204907.944613:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204907.944617:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204907.944618:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204907.944622:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204907.944624:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204907.944649:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204907.944653:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204907.944656:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204908.264506:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301035009, at 4301035520 00000100:00000040:1.0:1788204908.264506:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301035009, at 4301035520 00000100:00000001:3.0:1788204908.264506:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204908.264514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204908.264518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.264524:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204908.264526:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000010:0.0:1788204908.264528:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000001:3.0:1788204908.264528:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204908.264536:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204908.264541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.264545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204908.264557:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204908.712510:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204908.712518:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204908.712520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204908.712525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204908.712530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204908.712536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204908.712540:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204908.712544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204908.712548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204908.776524:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204908.776524:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204908.776531:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204908.776535:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204908.776535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204908.776542:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.776543:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204908.776551:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204908.776553:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204908.776556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204908.776560:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204908.776560:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.776568:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204908.776570:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204908.776572:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204908.776579:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204908.776584:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204908.776586:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204908.776593:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204908.776595:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.776597:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204908.776602:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204908.776609:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204908.776620:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6363 00000400:00000200:0.0:1788204908.776641:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204908.776648:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204908.776651:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204908.776655:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776658:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.776660:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776661:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204908.776665:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204908.776667:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204908.776669:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204908.776673:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204908.776674:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204908.776676:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204908.776679:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776681:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.776682:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776683:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204908.776686:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204908.776689:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776704:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204908.776706:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204908.776709:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204908.776712:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.776713:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204908.776716:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204908.776727:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204908.776730:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204908.776734:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204908.776735:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.776739:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204908.776741:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204908.776745:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204908.776765:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788204908.776772:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204908.776781:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204908.776793:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204908.776799:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204908.776804:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.776847:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204908.776851:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204908.776855:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204908.776857:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204908.776859:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204908.776863:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204908.776866:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6363 00000400:00000200:0.0:1788204908.776870:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204908.776874:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204908.776877:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204908.776894:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204908.776896:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204908.776898:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.776905:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204908.776905:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204908.776914:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204908.776915:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204908.776929:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204908.776932:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000040:3.0:1788204908.776932:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.776946:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.776948:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204908.776953:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204908.776957:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:3.0:1788204908.776957:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788204908.776965:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204908.776967:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788204908.776968:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204908.776974:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776976:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.776979:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.776984:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788204908.776985:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204908.776987:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204908.776994:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00100000:3.0:1788204908.776999:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205909632:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204908.777005:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204908.777011:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.777012:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204908.777013:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204908.777020:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204908.777023:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204908.777025:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204908.777031:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777034:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204908.777039:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000010:3.0:1788204908.777040:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 00000008:00000001:0.0:1788204908.777044:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.777046:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777047:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204908.777061:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:0.0:1788204908.777065:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000400:00000010:3.0:1788204908.777067:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000020:00000040:0.0:1788204908.777069:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204908.777075:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204908.777077:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777079:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788204908.777080:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 02000000:00000001:0.0:1788204908.777082:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204908.777087:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204908.777093:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000200:3.0:1788204908.777094:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909632, portal 25 00000100:00000001:0.0:1788204908.777097:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777101:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788204908.777119:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204908.777119:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000001:0.0:1788204908.777135:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204908.777138:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204908.777138:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204908.777147:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.777181:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204908.777188:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204908.777197:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000001:0.0:1788204908.777199:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204908.777204:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:3.0:1788204908.777207:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205909632, offset 0 00000100:00080000:0.0:1788204908.777209:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204908.777212:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204908.777214:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204908.777218:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204908.777224:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6363 00000400:00000200:3.0:1788204908.777232:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777250:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204908.777254:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204908.777267:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204908.777268:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:0.0:1788204908.777274:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204908.777281:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777281:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204908.777288:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.777291:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777294:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204908.777296:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788204908.777298:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204908.777300:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204908.777303:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204908.777305:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788204908.777306:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204908.777313:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204908.777315:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788204908.777317:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204908.777324:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777328:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58962:58962:256:4294967295] 192.168.201.151@tcp LPNI seq info [58942:58942:8:4294967295] 00000100:00000001:0.0:1788204908.777331:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204908.777340:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777344:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204908.777352:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204908.777352:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204908.777406:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777407:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204908.777411:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204908.777416:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204908.777423:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204908.777427:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777432:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:3.0:1788204908.777433:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000008:00100000:0.0:1788204908.777436:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204908.777442:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777444:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204908.777446:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000010:0.0:1788204908.777449:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204908.777453:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204908.777456:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.0:1788204908.777462:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204908.777463:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777468:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204908.777474:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204908.777477:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204908.777483:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000010:0.0:1788204908.777486:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788204908.777495:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204908.777500:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204908.777506:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204908.777507:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777513:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204908.777521:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204908.777523:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.777527:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301040130 00000100:00000001:0.0:1788204908.777542:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204908.777542:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f42a0. 00000100:00080000:0.0:1788204908.777545:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204908.777549:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204908.777551:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204908.777553:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204908.777554:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204908.777556:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204908.777562:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6363 00000100:00000001:3.0:1788204908.777567:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301040130 00000400:00000200:0.0:1788204908.777568:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:3.0:1788204908.777576:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4300. 00000100:00000001:3.0:1788204908.777582:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000400:00000200:0.0:1788204908.777584:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204908.777589:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204908.777593:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788204908.777596:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777603:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204908.777604:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204908.777606:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777613:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204908.777616:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204908.777617:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204908.777619:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204908.777622:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204908.777625:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788204908.777626:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204908.777637:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204908.777640:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204908.777643:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777644:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204908.777645:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788204908.777647:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204908.777651:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204908.777652:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000040:0.0:1788204908.777655:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204908.777660:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204908.777662:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204908.777663:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204908.777668:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204908.777670:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788204908.777671:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204908.777677:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777677:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204908.777679:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204908.777684:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000040:3.0:1788204908.777691:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204908.777698:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777705:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204908.777712:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204908.777717:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204908.777717:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788204908.777727:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204908.777727:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204908.777730:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777735:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204908.777737:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204908.777737:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204908.777741:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.777744:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204908.777749:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.777754:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909760:192.168.201.151@tcp:38:kworker.0 02000000:00000010:0.0:1788204908.777756:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204908.777764:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777765:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204908.777773:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1788204908.777775:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204908.777777:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.777788:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204908.777794:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 00000100:00000001:0.0:1788204908.777796:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788204908.777799:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.777802:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204908.777809:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204908.777813:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204908.777816:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5a50. 00000400:00000010:3.0:1788204908.777826:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204908.777838:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909760, portal 10 00000100:00000001:0.0:1788204908.777842:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777843:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204908.777847:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:0.0:1788204908.777855:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6373) 00000100:00000040:3.0:1788204908.777862:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.777872:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204908.777876:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204908.777879:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204908.777882:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205909760, offset 0 00000400:00000200:3.0:1788204908.777888:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777893:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.777900:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204908.777905:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204908.777908:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204908.777911:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204908.777914:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204908.777917:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58963:58963:255:4294967295] 192.168.201.151@tcp LPNI seq info [58943:58943:7:4294967295] 00000400:00000200:3.0:1788204908.777922:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.777931:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204908.777936:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204908.777940:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecb00. 00000800:00000200:3.0:1788204908.777944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204908.777949:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204908.777952:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204908.777956:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204908.777959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204908.777961:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301036187 00000100:00000001:3.0:1788204908.777964:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.777967:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301036187 00000100:00000001:3.0:1788204908.777969:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.777972:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204908.777974:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.777976:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204908.777981:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.777989:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.777991:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.777993:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204908.777995:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204908.777998:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.778001:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909888:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204908.778004:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204908.778007:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204908.778009:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.778011:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204908.778015:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204908.778017:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204908.778020:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204908.778023:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204908.778027:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204908.778031:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205909888, portal 4 00000100:00000001:3.0:1788204908.778033:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204908.778034:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204908.778041:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.778048:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204908.778051:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204908.778054:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4198. 00000100:00000200:3.0:1788204908.778056:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205909888, offset 0 00000400:00000200:3.0:1788204908.778061:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204908.778065:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.778070:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204908.778073:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204908.778075:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204908.778078:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204908.778081:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204908.778084:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58964:58964:254:4294967295] 192.168.201.151@tcp LPNI seq info [58944:58944:6:4294967295] 00000400:00000200:3.0:1788204908.778088:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.778096:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204908.778101:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204908.778119:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ece00. 00000800:00000200:3.0:1788204908.778123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204908.778127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204908.778130:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204908.778133:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204908.778137:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204908.778139:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301036073 00000100:00000001:3.0:1788204908.778141:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.778144:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301036073 00000100:00000001:3.0:1788204908.778146:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.778148:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204908.778151:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.778154:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.778156:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204908.778160:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204908.778162:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204908.778164:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204908.778166:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204908.778167:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204908.778172:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.778180:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.778182:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.778184:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204908.778186:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204908.778188:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.778191:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910016:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204908.778195:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204908.778198:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204908.778200:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204908.778201:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204908.778206:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204908.778208:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204908.778211:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204908.778214:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5f20. 00000400:00000010:3.0:1788204908.778218:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f44c8. 00000100:00000200:3.0:1788204908.778221:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910016, portal 4 00000100:00000001:3.0:1788204908.778224:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204908.778225:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204908.778231:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204908.778238:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204908.778240:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204908.778243:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4088. 00000100:00000200:3.0:1788204908.778246:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205910016, offset 0 00000400:00000200:3.0:1788204908.778251:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204908.778255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.778258:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204908.778261:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204908.778264:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204908.778267:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204908.778269:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204908.778273:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58965:58965:253:4294967295] 192.168.201.151@tcp LPNI seq info [58945:58945:5:4294967295] 00000400:00000200:3.0:1788204908.778277:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204908.778284:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204908.778289:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204908.778292:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204908.778296:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204908.778300:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204908.778303:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204908.778306:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204908.778309:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204908.778312:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301036079 00000100:00000001:3.0:1788204908.778314:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.778317:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301036079 00000100:00000001:3.0:1788204908.778319:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301040130 00000100:00000001:3.0:1788204908.778321:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204908.778323:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.778325:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.778327:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204908.778330:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204908.778332:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204908.778334:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204908.778335:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204908.778337:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.778339:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204908.968513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204908.968521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204908.968526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204908.968527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204908.968530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204908.968531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204908.968538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204908.968539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204908.968622:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204908.968628:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204908.968632:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204909.288483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204909.288489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204909.288491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204909.288495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204909.288497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204909.288500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204909.288503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204909.288505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204909.288507:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204909.736503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204909.736511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204909.736514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204909.736520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204909.736524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204909.736529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204909.736531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204909.736535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204909.736538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204909.800542:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204909.800551:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204909.800553:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204909.800559:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204909.800563:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204909.800566:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204909.800571:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204909.800574:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204909.800578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204909.800593:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204909.800599:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204909.800602:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204909.800612:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204909.800617:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204909.800621:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204909.800624:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204909.800627:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204909.800631:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204909.992528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204909.992535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204909.992541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204909.992542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204909.992544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204909.992545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204909.992548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204909.992549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204909.992569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204909.992572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204909.992574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204910.312500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204910.312512:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204910.312515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204910.312523:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204910.312528:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204910.312533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204910.312535:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204910.312538:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204910.312541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.760492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204910.760501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204910.760504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204910.760510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204910.760513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204910.760518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.760521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204910.760523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.760526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.824507:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204910.824514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204910.824517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.824523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204910.824528:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204910.824534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204910.824538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204910.824541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204910.824551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.824579:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204910.824585:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204910.824588:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204910.824592:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204910.824595:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204910.824599:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.824602:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204910.824604:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.824607:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204911.016526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204911.016534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204911.016538:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204911.016541:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016544:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204911.016546:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016547:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016550:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204911.016553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204911.016556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204911.016557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204911.016627:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204911.016631:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204911.016634:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204911.336495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204911.336505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204911.336507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204911.336512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204911.336515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204911.336520:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204911.336523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204911.336526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204911.336528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204911.784534:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204911.784545:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204911.784548:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204911.784555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204911.784560:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204911.784565:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204911.784568:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204911.784571:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204911.784575:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204911.848469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204911.848476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204911.848478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204911.848487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204911.848490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204911.848493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.848495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204911.848497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.848500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204911.848667:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204911.848674:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204911.848676:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204911.848680:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204911.848683:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204911.848686:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204911.848689:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204911.848691:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204911.848693:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204911.848813:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:0.0:1788204911.848864:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204911.848874:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204911.848879:0:2334:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:0.0:1788204911.848884:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204911.848889:0:2334:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788204911.848895:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204911.848899:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:0.0:1788204911.848905:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204911.848910:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204911.848917:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:0.0:1788204911.848920:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:0.0:1788204911.848925:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204911.848935:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.848970:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204911.848976:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.848986:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204911.848989:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204911.849000:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204911.849003:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecb00. 00000400:00000200:0.0:1788204911.849006:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204911.849010:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204911.849014:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:0.0:1788204911.849016:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788204911.849018:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204911.849023:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849032:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204911.849036:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849043:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204911.849045:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204911.849084:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204911.849087:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ece00. 00000400:00000200:0.0:1788204911.849091:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204911.849095:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204911.849098:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4198 00000400:00000010:0.0:1788204911.849100:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4198. 00000100:00000001:0.0:1788204911.849103:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204911.849108:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849117:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204911.849120:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849128:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204911.849131:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204911.849137:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:0.0:1788204911.849140:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:0.0:1788204911.849143:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204911.849147:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204911.849150:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4088 00000400:00000010:0.0:1788204911.849153:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4088. 00000100:00000001:0.0:1788204911.849155:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204911.849160:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849169:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204911.849173:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204911.849180:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204911.849182:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204911.849413:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204911.849418:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204911.849422:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204911.849432:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204908/real 1788204911] req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204911.849447:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204911.849450:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204911.849453:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5f20. 00000400:00000200:3.0:1788204911.849460:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f44c8 00000400:00000010:3.0:1788204911.849463:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f44c8. 00000100:00000001:3.0:1788204911.849468:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204911.849476:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204911.849486:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849493:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204911.849497:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849502:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849505:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204911.849514:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849522:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204911.849529:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849537:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204911.849608:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849611:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849613:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849616:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204911.849619:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204911.849633:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204911.849640:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204911.849643:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204911.849649:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204911.849658:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910016:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204911.849664:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204911.849668:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204908/real 1788204911] req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204911.849684:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204911.849686:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204911.849688:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5a50. 00000400:00000200:3.0:1788204911.849692:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204911.849694:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204911.849697:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204911.849700:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204911.849710:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849717:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204911.849718:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849720:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849722:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204911.849726:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849734:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204911.849738:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849745:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204911.849748:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849749:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849751:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849752:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204911.849755:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204911.849758:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204911.849762:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204911.849765:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204911.849770:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204911.849778:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909760:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204911.849782:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204911.849786:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204908/real 1788204911] req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204911.849794:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204911.849796:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204911.849797:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204911.849801:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204911.849802:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204911.849805:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204911.849808:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204911.849818:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849824:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204911.849826:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849827:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849829:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204911.849833:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849840:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204911.849844:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849851:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204911.849853:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849855:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849857:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849858:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204911.849861:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204911.849864:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204911.849867:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204911.849876:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204911.849881:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204911.849889:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205909888:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204911.849893:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204911.849897:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204908/real 1788204911] req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204911.849904:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204911.849906:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204911.849908:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204911.849910:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204911.849913:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204911.849915:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204911.849918:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204911.849927:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849935:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204911.849936:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849938:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849940:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204911.849943:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849956:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204911.849960:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.849967:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204911.849969:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849971:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204911.849973:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.849975:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204911.849976:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204911.849979:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204911.849982:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204911.849985:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204911.849989:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204911.849996:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205909632:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204911.850001:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850003:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204911.850007:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205910016/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.850021:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204911.850025:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204911.850029:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204911.850033:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204911.850036:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204911.850038:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204911.850041:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204911.850043:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:3.0:1788204911.850047:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204911.850062:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204911.850065:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204911.850069:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204911.850070:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850072:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204911.850088:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205909760/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.850095:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204911.850096:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204911.850098:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204911.850102:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204911.850103:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204911.850104:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204911.850107:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204911.850108:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788204911.850112:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204911.850113:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204911.850115:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204911.850117:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204911.850119:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850121:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204911.850125:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205909888/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.850132:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204911.850133:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204911.850135:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204911.850138:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204911.850139:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204911.850141:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204911.850143:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204911.850144:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788204911.850147:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204911.850148:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204911.850150:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204911.850152:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204911.850154:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850156:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204911.850160:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205909632/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204911.850167:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204911.850168:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204911.850169:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204911.850172:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204911.850173:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204911.850174:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204911.850176:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204911.850178:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788204911.850181:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204911.850182:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204911.850183:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204911.850185:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204911.850186:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850189:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.850197:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204911.850199:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204911.850202:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204911.850204:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204912.040496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204912.040503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204912.040508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204912.040510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204912.040513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204912.040515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204912.040518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204912.040520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204912.040537:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204912.040540:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204912.040543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204912.360481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204912.360488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204912.360490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204912.360496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204912.360499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204912.360504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204912.360507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204912.360510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204912.360513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.808444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204912.808452:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204912.808454:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204912.808460:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204912.808463:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204912.808466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204912.808469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204912.808471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204912.808474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204912.872435:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204912.872441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204912.872443:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204912.872447:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204912.872450:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204912.872453:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204912.872455:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204912.872457:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204912.872459:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.872508:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204912.872512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204912.872514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204912.872517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204912.872520:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204912.872522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204912.872524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204912.872526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204912.872528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204913.064482:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204913.064485:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204913.064489:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064494:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204913.064496:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064498:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064501:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204913.064507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204913.064512:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204913.064514:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204913.064515:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064517:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204913.064518:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064520:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064522:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204913.064527:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204913.064528:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204913.064530:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064532:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204913.064533:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064535:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204913.064536:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204913.064600:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204913.064603:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204913.064606:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204913.384497:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301040130, at 4301040640 00000100:00000010:1.0:1788204913.384513:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000040:0.0:1788204913.384525:0:48471:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301040130, at 4301040640 00000100:00000010:0.0:1788204913.384535:0:48471:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000001:3.0:1788204913.384552:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204913.384557:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204913.384560:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204913.384565:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204913.384568:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204913.384573:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.384576:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204913.384580:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.384584:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204913.832476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204913.832481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204913.832483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204913.832487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204913.832490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204913.832493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204913.832495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204913.832497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204913.832499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.896428:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204913.896434:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204913.896437:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204913.896443:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204913.896447:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204913.896451:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.896454:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204913.896456:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204913.896457:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204913.896464:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204913.896464:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.896467:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204913.896471:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204913.896475:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204913.896480:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6368 00000400:00000200:0.0:1788204913.896488:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204913.896493:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204913.896497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204913.896502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204913.896506:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204913.896506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204913.896510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000040:0.0:1788204913.896513:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204913.896514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204913.896518:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204913.896519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204913.896522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1788204913.896523:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204913.896524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204913.896527:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788204913.896528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204913.896533:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204913.896536:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204913.896538:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204913.896540:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204913.896543:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204913.896544:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204913.896547:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896548:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204913.896549:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896551:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204913.896553:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204913.896555:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896556:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204913.896558:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204913.896561:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204913.896563:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.896564:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204913.896566:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204913.896582:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204913.896586:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204913.896589:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204913.896591:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.896595:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204913.896597:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204913.896601:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204913.896609:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788204913.896616:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204913.896625:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204913.896634:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204913.896639:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204913.896644:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204913.896668:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204913.896671:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204913.896675:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204913.896677:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204913.896679:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204913.896682:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204913.896685:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6368 00000400:00000200:0.0:1788204913.896696:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204913.896701:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204913.896704:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204913.896707:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788204913.896708:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204913.896713:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204913.896714:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204913.896719:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204913.896720:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896725:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204913.896729:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:3.0:1788204913.896730:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788204913.896731:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204913.896735:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204913.896739:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788204913.896744:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204913.896751:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204913.896752:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204913.896755:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204913.896761:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204913.896763:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204913.896765:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788204913.896768:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788204913.896771:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204913.896774:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.896777:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788204913.896782:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205910144:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788204913.896784:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204913.896792:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788204913.896793:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896799:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.896799:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204913.896802:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204913.896804:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.896809:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788204913.896811:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204913.896816:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204913.896816:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 00000100:00000001:0.0:1788204913.896823:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204913.896823:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788204913.896831:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204913.896834:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:0.0:1788204913.896839:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204913.896841:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204913.896844:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204913.896851:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000010:0.0:1788204913.896852:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788204913.896858:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000200:3.0:1788204913.896861:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910144, portal 25 02000000:00000001:0.0:1788204913.896862:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204913.896865:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.896868:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204913.896869:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204913.896872:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 02000000:00000001:0.0:1788204913.896873:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204913.896877:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204913.896884:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204913.896886:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204913.896893:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.896903:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788204913.896905:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204913.896910:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204913.896917:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000001:0.0:1788204913.896920:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204913.896924:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205910144, offset 0 00000100:00000001:0.0:1788204913.896925:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204913.896933:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204913.896937:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204913.896950:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.896952:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204913.896956:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204913.896962:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788204913.896963:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788204913.896965:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204913.896969:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204913.896973:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788204913.896974:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204913.896980:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6368 00000400:00000200:3.0:1788204913.896982:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788204913.896989:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.896991:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204913.896997:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788204913.897000:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204913.897005:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204913.897006:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58966:58966:256:4294967295] 192.168.201.151@tcp LPNI seq info [58946:58946:8:4294967295] 00000400:00000200:3.0:1788204913.897029:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788204913.897031:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897036:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204913.897042:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897047:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204913.897050:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:0.0:1788204913.897052:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204913.897056:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204913.897061:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:3.0:1788204913.897061:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788204913.897065:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204913.897070:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:3.0:1788204913.897071:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000020:00000001:0.0:1788204913.897073:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788204913.897081:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788204913.897082:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204913.897085:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204913.897089:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00000040:0.0:1788204913.897092:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897098:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.1:1788204913.897102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204913.897104:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.1:1788204913.897109:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00080000:0.0:1788204913.897112:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897118:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204913.897119:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788204913.897124:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1788204913.897125:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 02000000:00000001:0.0:1788204913.897130:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204913.897135:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897137:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204913.897140:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204913.897143:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.897145:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204913.897146:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.897155:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000010:0.0:1788204913.897157:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204913.897162:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788204913.897163:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 02000000:00000001:0.0:1788204913.897167:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788204913.897168:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 02000000:00000001:0.0:1788204913.897172:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897175:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204913.897176:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204913.897179:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204913.897184:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204913.897184:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204913.897192:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788204913.897199:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897202:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1788204913.897209:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204913.897219:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204913.897221:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204913.897227:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204913.897234:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204913.897237:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204913.897240:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204913.897241:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204913.897243:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204913.897246:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204913.897249:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6368 00000400:00000200:0.0:1788204913.897253:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204913.897255:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204913.897258:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204913.897261:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897262:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204913.897264:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897266:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204913.897269:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204913.897270:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204913.897272:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204913.897274:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204913.897276:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204913.897278:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204913.897280:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897282:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204913.897284:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897286:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204913.897288:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204913.897291:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204913.897293:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204913.897295:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204913.897298:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204913.897300:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897302:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204913.897304:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000040:3.0:1788204913.897306:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204913.897307:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897312:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204913.897318:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204913.897323:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204913.897329:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204913.897332:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897337:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204913.897340:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204913.897342:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204913.897346:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:3.0:1788204913.897349:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897351:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897354:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1788204913.897360:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204913.897388:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.897395:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204913.897399:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301045250 00000100:00000010:3.0:1788204913.897405:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c00. 00000100:00000001:3.0:1788204913.897414:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204913.897424:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204913.897425:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301045250 00000100:00000001:0.0:1788204913.897428:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204913.897433:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f42a0. 00000100:00000040:0.0:1788204913.897438:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897439:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204913.897447:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204913.897453:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204913.897455:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897459:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788204913.897462:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6378) 00000100:00100000:3.0:1788204913.897469:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204913/real 1788204913] req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204913.897482:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204913.897484:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204913.897487:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204913.897490:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204913.897492:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204913.897495:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204913.897499:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204913.897508:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897515:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204913.897516:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897519:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.897527:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204913.897533:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897541:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204913.897546:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897552:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204913.897555:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897557:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.897558:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897560:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204913.897562:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204913.897565:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204913.897568:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204913.897571:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204913.897575:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204913.897582:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205910144:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204913.897586:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897588:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.897591:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205910144/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897597:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204913.897600:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204913.897603:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204913.897606:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204913.897609:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204913.897610:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204913.897612:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204913.897615:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:3.0:1788204913.897619:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204913.897622:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204913.897623:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204913.897626:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204913.897628:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897630:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897637:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204913.897638:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897641:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204913.897642:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204913.897644:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204913.897648:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897655:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204913.897657:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.897659:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204913.897661:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204913.897664:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204913.897667:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910272:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204913.897670:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204913.897673:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204913.897674:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.897676:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204913.897680:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204913.897681:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.897686:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204913.897689:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204913.897693:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204913.897696:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910272, portal 10 00000100:00000001:3.0:1788204913.897698:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204913.897699:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204913.897706:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897712:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204913.897715:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204913.897718:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204913.897720:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205910272, offset 0 00000400:00000200:3.0:1788204913.897725:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204913.897729:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.897734:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204913.897737:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204913.897740:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204913.897743:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204913.897745:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204913.897748:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58967:58967:256:4294967295] 192.168.201.151@tcp LPNI seq info [58947:58947:8:4294967295] 00000400:00000200:3.0:1788204913.897752:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.897758:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204913.897764:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.897767:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204913.897771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204913.897774:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204913.897779:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204913.897782:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204913.897786:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.897788:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204913.897792:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204913.897796:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204913.897799:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204913.897801:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204913.897804:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204913.897808:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897817:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.897820:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897828:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897829:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204913.897833:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204913.897837:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204913.897841:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204913.897843:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301041307 00000100:00000001:3.0:1788204913.897846:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.897849:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301041307 00000100:00000001:3.0:1788204913.897851:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.897853:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204913.897855:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.897857:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204913.897861:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204913/real 1788204913] req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204913.897869:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204913.897870:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204913.897872:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204913.897875:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204913.897877:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204913.897879:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204913.897883:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204913.897891:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897899:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204913.897900:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897902:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.897903:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204913.897907:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897912:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204913.897917:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897924:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204913.897927:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897928:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.897929:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.897931:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204913.897934:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204913.897937:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204913.897940:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204913.897943:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204913.897947:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204913.897955:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910272:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204913.897959:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204913.897962:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.897969:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204913.897971:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.897973:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204913.897975:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204913.897977:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204913.897981:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910400:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204913.897984:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204913.897986:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204913.897988:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.897990:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204913.897994:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204913.897996:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.897999:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204913.898002:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204913.898006:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204913.898009:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910400, portal 4 00000100:00000001:3.0:1788204913.898011:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204913.898012:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204913.898033:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898041:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204913.898043:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204913.898046:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204913.898049:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205910400, offset 0 00000400:00000200:3.0:1788204913.898053:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204913.898058:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.898062:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204913.898065:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204913.898068:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204913.898070:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204913.898073:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204913.898076:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58968:58968:256:4294967295] 192.168.201.151@tcp LPNI seq info [58948:58948:8:4294967295] 00000400:00000200:3.0:1788204913.898081:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.898088:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204913.898094:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.898098:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204913.898102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204913.898105:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204913.898109:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204913.898112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204913.898116:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.898118:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204913.898122:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204913.898126:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204913.898129:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204913.898131:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204913.898134:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204913.898137:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898146:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.898149:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898156:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.898158:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204913.898162:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204913.898165:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204913.898168:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204913.898170:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301041193 00000100:00000001:3.0:1788204913.898172:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.898175:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301041193 00000100:00000001:3.0:1788204913.898177:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.898180:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204913.898182:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.898184:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204913.898188:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204913/real 1788204913] req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204913.898195:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204913.898197:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204913.898199:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204913.898201:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204913.898202:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204913.898205:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204913.898208:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204913.898216:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898226:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204913.898228:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898230:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.898231:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204913.898235:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898240:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204913.898244:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898251:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204913.898253:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898255:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.898256:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898258:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204913.898260:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204913.898263:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204913.898266:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204913.898268:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204913.898272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204913.898279:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910400:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204913.898283:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204913.898286:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898292:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204913.898294:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.898297:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204913.898298:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204913.898301:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204913.898303:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910528:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204913.898307:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204913.898309:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204913.898311:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204913.898312:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204913.898316:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204913.898318:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204913.898321:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204913.898324:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204913.898327:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204913.898330:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910528, portal 4 00000100:00000001:3.0:1788204913.898331:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204913.898333:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204913.898339:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898344:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204913.898346:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204913.898348:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204913.898350:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205910528, offset 0 00000400:00000200:3.0:1788204913.898354:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204913.898358:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.898390:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204913.898394:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204913.898396:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204913.898399:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204913.898401:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204913.898405:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58969:58969:256:4294967295] 192.168.201.151@tcp LPNI seq info [58949:58949:8:4294967295] 00000400:00000200:3.0:1788204913.898408:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204913.898414:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204913.898420:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.898424:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204913.898428:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204913.898430:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204913.898434:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204913.898437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204913.898454:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204913.898457:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204913.898460:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204913.898463:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204913.898466:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204913.898468:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204913.898471:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204913.898474:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898481:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.898484:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898490:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.898492:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204913.898495:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204913.898498:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204913.898501:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204913.898504:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301041200 00000100:00000001:3.0:1788204913.898506:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.898508:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301041200 00000100:00000001:3.0:1788204913.898510:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301045250 00000100:00000001:3.0:1788204913.898512:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204913.898514:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.898516:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204913.898520:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204913/real 1788204913] req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204913.898534:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204913.898535:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204913.898537:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204913.898539:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204913.898541:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204913.898542:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204913.898546:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204913.898555:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898562:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204913.898564:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898566:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.898567:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204913.898571:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898577:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204913.898581:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898587:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204913.898589:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898591:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204913.898592:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898594:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204913.898596:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204913.898598:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204913.898601:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204913.898604:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204913.898609:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204913.898616:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910528:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204913.898620:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898622:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.898625:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205910272/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898632:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204913.898634:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204913.898635:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204913.898639:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204913.898640:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204913.898642:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204913.898644:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204913.898648:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788204913.898651:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204913.898652:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204913.898654:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204913.898656:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204913.898657:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898659:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.898662:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205910400/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898668:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204913.898669:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204913.898670:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204913.898673:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204913.898674:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204913.898675:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204913.898677:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204913.898679:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788204913.898681:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204913.898683:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204913.898684:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204913.898686:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204913.898687:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898689:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204913.898692:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205910528/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204913.898697:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204913.898698:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204913.898700:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204913.898703:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204913.898704:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204913.898705:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204913.898707:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204913.898708:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788204913.898711:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204913.898712:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204913.898713:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204913.898715:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204913.898716:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898718:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.898723:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204913.898724:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.898726:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204913.898728:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204914.088532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204914.088538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204914.088542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204914.088543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204914.088545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204914.088546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204914.088548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204914.088550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204914.088567:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204914.088569:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204914.088571:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204914.408468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204914.408477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204914.408479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204914.408484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204914.408488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204914.408491:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204914.408495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204914.408508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204914.408512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.856461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204914.856467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204914.856469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204914.856473:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204914.856475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204914.856478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204914.856481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204914.856483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204914.856485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204914.920445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204914.920452:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204914.920454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204914.920458:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204914.920462:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204914.920465:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204914.920467:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204914.920469:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204914.920471:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.920519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204914.920523:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204914.920524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204914.920527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204914.920529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204914.920530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204914.920532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204914.920533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204914.920535:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204915.112498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204915.112505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204915.112513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204915.112514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204915.112518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204915.112519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204915.112523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204915.112524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204915.112589:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204915.112594:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204915.112599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204915.432469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204915.432475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204915.432478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204915.432483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204915.432486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204915.432489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.432491:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204915.432493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.432495:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.880472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204915.880480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.880483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.880489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204915.880493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204915.880497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.880501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.880503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.880507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.944467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204915.944467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204915.944478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204915.944478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.944483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204915.944483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.944491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204915.944491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204915.944498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204915.944499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204915.944502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.944504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.944506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204915.944510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.944512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.944518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.944522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.944525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204916.136515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204916.136522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204916.136527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204916.136529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204916.136532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204916.136533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204916.136543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204916.136544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204916.136562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204916.136564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204916.136567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204916.456495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204916.456509:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204916.456512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204916.456518:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204916.456522:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204916.456531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204916.456534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204916.456537:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204916.456541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204916.904432:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204916.904450:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204916.904452:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204916.904460:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204916.904464:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204916.904468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.904470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204916.904472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.904475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.968485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204916.968486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204916.968495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204916.968496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204916.968499:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.968500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204916.968508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204916.968509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204916.968516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204916.968516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204916.968522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204916.968522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.968527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204916.968530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.968533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.968534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204916.968537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204916.968540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204917.160496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204917.160505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204917.160518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204917.160519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204917.160521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204917.160522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204917.160524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204917.160525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204917.160581:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204917.160587:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204917.160591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204917.480471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204917.480481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204917.480484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204917.480491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204917.480495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204917.480500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204917.480503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204917.480506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204917.480510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204917.928484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204917.928492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204917.928494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204917.928500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204917.928503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204917.928506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.928508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204917.928510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.928513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204917.992445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204917.992449:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204917.992454:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204917.992458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204917.992459:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204917.992460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204917.992469:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204917.992469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204917.992477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204917.992477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204917.992484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204917.992484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.992492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204917.992497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.992501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204917.992501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204917.992504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204917.992508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204918.184570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204918.184579:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204918.184587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204918.184588:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204918.184592:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204918.184593:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204918.184596:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204918.184598:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204918.184622:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204918.184625:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204918.184629:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204918.504470:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301045250, at 4301045760 00000100:00000040:1.0:1788204918.504470:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301045250, at 4301045760 00000100:00000001:3.0:1788204918.504470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204918.504476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204918.504479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204918.504483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204918.504486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204918.504487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000010:0.0:1788204918.504490:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000001:3.0:1788204918.504490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204918.504494:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204918.504497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204918.504500:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204918.952501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204918.952513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204918.952515:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204918.952523:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204918.952527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204918.952532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204918.952536:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204918.952539:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204918.952542:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204919.016476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204919.016476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204919.016485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204919.016485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204919.016487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204919.016488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204919.016495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204919.016495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204919.016501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204919.016502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204919.016508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204919.016510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204919.016511:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204919.016514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.016519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204919.016521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.016523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204919.016541:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204919.016545:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204919.016547:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204919.016549:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204919.016553:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204919.016560:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6373 00000400:00000200:3.0:1788204919.016569:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204919.016575:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204919.016579:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204919.016582:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016585:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016588:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016590:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204919.016595:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204919.016597:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204919.016600:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204919.016603:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204919.016606:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204919.016608:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204919.016611:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016613:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016615:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016617:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204919.016621:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204919.016624:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016627:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204919.016629:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204919.016632:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204919.016634:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016637:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204919.016639:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204919.016648:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0380. 00000020:00000040:3.0:1788204919.016651:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204919.016655:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204919.016657:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016659:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204919.016662:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204919.016665:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204919.016673:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22800. 00000100:00000001:3.0:1788204919.016679:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204919.016687:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204919.016698:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204919.016703:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204919.016707:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2501e67] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204919.016732:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204919.016735:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204919.016738:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204919.016739:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204919.016740:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204919.016742:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204919.016744:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6373 00000400:00000200:3.0:1788204919.016747:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204919.016748:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204919.016750:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204919.016752:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016754:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016755:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016757:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204919.016760:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204919.016762:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204919.016764:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204919.016767:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788204919.016769:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788204919.016771:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204919.016775:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204919.016777:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204919.016777:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016782:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204919.016785:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204919.016786:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204919.016789:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1788204919.016790:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788204919.016795:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016799:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204919.016800:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204919.016801:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204919.016807:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204919.016811:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016814:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204919.016818:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788204919.016819:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788204919.016820:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788204919.016825:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.016827:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788204919.016827:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.016830:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788204919.016832:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3100. 02000000:00000001:2.0:1788204919.016833:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204919.016836:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00100000:2.0:1788204919.016839:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205910656:192.168.201.151@tcp:250:kworker.0 02000000:00000001:3.0:1788204919.016840:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204919.016842:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016844:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788204919.016847:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204919.016847:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204919.016849:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.016853:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:3.0:1788204919.016853:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204919.016856:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016857:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.016859:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788204919.016864:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1788204919.016865:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204919.016870:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.016876:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788204919.016880:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204919.016881:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204919.016886:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e74a292a] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788204919.016887:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204919.016897:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000001:3.0:1788204919.016902:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204919.016905:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1788204919.016907:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910656, portal 25 00000100:00080000:3.0:1788204919.016908:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204919.016911:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204919.016912:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788204919.016913:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204919.016915:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204919.016917:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6373 00000100:00000001:2.0:1788204919.016921:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300822400 : -113511408729216 : ffff98c310ce0380) 00000400:00000200:3.0:1788204919.016921:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204919.016924:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204919.016941:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204919.016943:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016944:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016947:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.016948:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:2.0:1788204919.016951:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204919.016952:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204919.016954:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788204919.016958:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204919.016961:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788204919.016965:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:2.0:1788204919.016968:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788204919.016969:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204919.016975:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:3.0:1788204919.016977:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:2.0:1788204919.016980:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204919.016981:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.016986:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:2.0:1788204919.016989:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205910656, offset 0 00000100:00000001:3.0:1788204919.016989:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204919.016996:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788204919.016998:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204919.017001:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204919.017004:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204919.017005:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788204919.017007:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204919.017010:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.017011:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204919.017013:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788204919.017015:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.017016:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017018:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:3.0:1788204919.017021:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2d80. 00000020:00000040:3.0:1788204919.017024:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788204919.017026:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204919.017028:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017030:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:3.0:1788204919.017030:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204919.017033:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204919.017035:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204919.017039:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20400. 00000400:00000200:2.0:1788204919.017042:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204919.017042:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204919.017048:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204919.017054:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204919.017056:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017060:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:3.0:1788204919.017061:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001291af88] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204919.017068:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204919.017070:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788204919.017074:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1788204919.017074:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204919.017077:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204919.017079:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204919.017083:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.017083:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204919.017087:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6373 00000400:00000200:2.0:1788204919.017092:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58970:58970:256:4294967295] 192.168.201.151@tcp LPNI seq info [58950:58950:8:4294967295] 00000400:00000200:2.0:1788204919.017098:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204919.017104:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204919.017111:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204919.017116:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:3.0:1788204919.017116:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204919.017123:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.017127:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1788204919.017132:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:3.0:1788204919.017132:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.017138:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204919.017139:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204919.017140:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788204919.017141:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:2.0:1788204919.017143:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000020:00000040:3.0:1788204919.017143:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788204919.017147:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788204919.017150:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788204919.017152:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204919.017153:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204919.017157:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204919.017158:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204919.017159:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204919.017162:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788204919.017165:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:2.0:1788204919.017166:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1788204919.017166:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204919.017168:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204919.017170:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204919.017173:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.017174:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204919.017176:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788204919.017179:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204919.017179:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204919.017183:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1788204919.017186:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000100:00000010:3.0:1788204919.017188:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3480. 00000020:00000040:3.0:1788204919.017193:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000800:00000100:2.0:1788204919.017194:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788204919.017198:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:2.0:1788204919.017201:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 02000000:00000001:3.0:1788204919.017201:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.017206:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204919.017208:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204919.017210:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017212:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:3.0:1788204919.017214:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23000. 00000400:00000200:2.0:1788204919.017220:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788204919.017220:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204919.017229:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204919.017230:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000001:3.0:1788204919.017238:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204919.017241:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788204919.017243:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000040:3.0:1788204919.017244:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133b1fcc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204919.017252:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017253:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788204919.017256:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6383) 00000100:00000200:2.0:1788204919.017261:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017273:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.017278:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017291:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017294:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204919.017301:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017306:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204919.017309:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301050369 00000100:00000010:2.0:1788204919.017315:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1840. 00000100:00000001:2.0:1788204919.017324:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204919.017336:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301050369 00000100:00000010:2.0:1788204919.017340:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1b40. 00000100:00000001:2.0:1788204919.017343:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204919.017347:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204919.017349:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017352:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204919.017357:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204919.017400:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204919.017402:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204919.017404:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204919.017407:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204919.017409:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204919.017412:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204919.017417:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204919.017425:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017432:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204919.017434:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017436:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017439:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204919.017443:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017450:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204919.017455:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017462:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204919.017465:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017467:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017468:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017471:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204919.017474:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204919.017476:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204919.017481:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204919.017483:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204919.017487:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204919.017495:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205910656:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204919.017499:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017501:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.017503:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0380 x1875062205910656/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017521:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204919.017523:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204919.017525:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204919.017528:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204919.017530:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204919.017531:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204919.017532:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204919.017535:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:2.0:1788204919.017538:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204919.017540:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204919.017541:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0380. 00000100:00000001:2.0:1788204919.017543:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204919.017544:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017546:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017553:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204919.017554:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017557:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204919.017558:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204919.017559:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204919.017563:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017570:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204919.017571:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.017573:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204919.017575:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204919.017577:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204919.017580:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910784:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204919.017584:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204919.017586:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204919.017587:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.017589:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204919.017592:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204919.017594:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017599:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204919.017601:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204919.017605:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204919.017607:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910784, portal 10 00000100:00000001:2.0:1788204919.017609:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204919.017610:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:2.0:1788204919.017615:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017620:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204919.017621:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204919.017623:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204919.017624:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205910784, offset 0 00000400:00000200:2.0:1788204919.017626:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204919.017629:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.017632:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204919.017634:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204919.017636:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204919.017637:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204919.017639:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204919.017641:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58971:58971:256:4294967295] 192.168.201.151@tcp LPNI seq info [58951:58951:8:4294967295] 00000400:00000200:2.0:1788204919.017644:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.017648:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204919.017650:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.017652:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204919.017655:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204919.017657:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204919.017660:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204919.017662:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204919.017671:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.017672:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204919.017674:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204919.017676:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204919.017679:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204919.017680:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204919.017682:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204919.017684:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017689:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.017691:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017695:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017697:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204919.017699:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204919.017702:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204919.017704:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204919.017705:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301046427 00000100:00000001:2.0:1788204919.017707:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.017709:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301046427 00000100:00000001:2.0:1788204919.017710:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.017711:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204919.017713:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017714:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204919.017716:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204919.017720:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204919.017721:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204919.017722:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204919.017724:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204919.017725:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204919.017726:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204919.017728:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204919.017732:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017740:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204919.017741:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017742:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017743:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204919.017745:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017749:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204919.017751:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017754:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204919.017756:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017756:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017757:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017758:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204919.017760:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204919.017761:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204919.017763:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204919.017765:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204919.017767:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204919.017771:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910784:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204919.017773:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204919.017775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017778:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204919.017779:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.017781:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204919.017782:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204919.017783:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204919.017785:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910912:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204919.017787:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204919.017788:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204919.017789:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.017790:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204919.017793:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204919.017795:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.017798:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204919.017801:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204919.017804:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204919.017806:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205910912, portal 4 00000100:00000001:2.0:1788204919.017808:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204919.017809:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:2.0:1788204919.017815:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017821:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204919.017824:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204919.017826:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204919.017828:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205910912, offset 0 00000400:00000200:2.0:1788204919.017832:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204919.017836:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.017839:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204919.017842:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204919.017844:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204919.017847:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204919.017848:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204919.017851:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58972:58972:256:4294967295] 192.168.201.151@tcp LPNI seq info [58952:58952:8:4294967295] 00000400:00000200:2.0:1788204919.017855:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.017860:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204919.017865:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.017869:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204919.017873:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204919.017875:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204919.017880:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204919.017883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204919.017886:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.017888:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204919.017891:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204919.017894:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204919.017897:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204919.017898:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204919.017900:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204919.017902:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017907:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.017909:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017913:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017914:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204919.017916:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204919.017919:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204919.017921:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204919.017922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301046313 00000100:00000001:2.0:1788204919.017924:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.017939:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301046313 00000100:00000001:2.0:1788204919.017941:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.017943:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204919.017945:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.017946:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204919.017949:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204919.017954:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204919.017955:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204919.017957:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204919.017958:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204919.017959:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204919.017961:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204919.017962:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204919.017968:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017971:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204919.017972:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017973:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017974:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204919.017975:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017979:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204919.017981:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.017986:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204919.017987:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017988:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.017989:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.017990:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204919.017991:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204919.017993:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204919.017994:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204919.017995:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204919.017998:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204919.018003:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205910912:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204919.018005:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204919.018007:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018010:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204919.018011:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.018012:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204919.018013:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204919.018014:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204919.018016:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911040:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204919.018018:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204919.018019:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204919.018020:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204919.018021:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204919.018024:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204919.018025:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204919.018028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204919.018030:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204919.018033:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204919.018036:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911040, portal 4 00000100:00000001:2.0:1788204919.018038:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204919.018039:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:2.0:1788204919.018045:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018051:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204919.018053:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204919.018055:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204919.018058:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205911040, offset 0 00000400:00000200:2.0:1788204919.018063:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204919.018067:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.018070:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204919.018073:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204919.018075:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204919.018078:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204919.018080:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204919.018083:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58973:58973:256:4294967295] 192.168.201.151@tcp LPNI seq info [58953:58953:8:4294967295] 00000400:00000200:2.0:1788204919.018087:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204919.018093:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204919.018097:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.018101:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204919.018105:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204919.018108:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1788204919.018112:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204919.018115:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1788204919.018119:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204919.018120:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204919.018123:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204919.018126:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204919.018129:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204919.018130:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204919.018133:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204919.018135:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018140:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.018142:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018146:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.018147:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204919.018149:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204919.018151:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204919.018153:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204919.018155:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301046319 00000100:00000001:2.0:1788204919.018157:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.018158:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301046319 00000100:00000001:2.0:1788204919.018159:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301050369 00000100:00000001:2.0:1788204919.018160:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204919.018162:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.018163:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204919.018165:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204919.018169:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204919.018170:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204919.018171:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204919.018172:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204919.018174:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204919.018175:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204919.018177:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204919.018182:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018185:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204919.018186:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018186:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.018187:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204919.018189:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018192:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204919.018194:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018198:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204919.018200:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018201:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204919.018201:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018203:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204919.018204:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204919.018206:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204919.018207:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204919.018208:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204919.018210:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204919.018214:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911040:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204919.018216:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018218:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.018220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205910784/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018224:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204919.018225:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204919.018226:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788204919.018227:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204919.018228:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204919.018229:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204919.018230:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204919.018231:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:2.0:1788204919.018233:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204919.018234:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204919.018235:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:2.0:1788204919.018237:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204919.018238:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018239:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.018240:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205910912/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018248:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204919.018249:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204919.018250:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:2.0:1788204919.018252:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204919.018252:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204919.018253:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204919.018254:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204919.018255:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204919.018256:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204919.018256:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204919.018257:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:2.0:1788204919.018259:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204919.018259:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018260:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204919.018262:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205911040/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204919.018265:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204919.018266:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204919.018267:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788204919.018269:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204919.018270:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204919.018271:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204919.018272:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204919.018273:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204919.018275:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204919.018276:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204919.018277:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:2.0:1788204919.018279:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204919.018280:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018281:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204919.018284:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204919.018286:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204919.018288:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204919.018289:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204919.208460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204919.208467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204919.208476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204919.208477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204919.208481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204919.208482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204919.208485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204919.208485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204919.208560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204919.208567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204919.208571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204919.528519:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204919.528532:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204919.528534:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204919.528541:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204919.528544:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204919.528548:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.528550:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204919.528553:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.528555:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204919.976500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204919.976509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204919.976511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204919.976516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204919.976518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204919.976522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204919.976524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204919.976526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204919.976529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.040494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204920.040494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204920.040505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204920.040505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204920.040509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204920.040509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.040518:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204920.040518:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204920.040524:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204920.040525:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204920.040529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204920.040530:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.040534:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.040537:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.040540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204920.040541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204920.040544:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204920.040548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204920.232562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204920.232570:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204920.232572:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204920.232575:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232577:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204920.232580:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232581:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232583:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232585:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204920.232586:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204920.232588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204920.232589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204920.232604:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204920.232606:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204920.232608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204920.552480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204920.552491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204920.552494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204920.552502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204920.552507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204920.552513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204920.552517:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204920.552520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204920.552523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.000520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204921.000529:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204921.000532:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204921.000538:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204921.000547:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204921.000555:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204921.000557:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204921.000560:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204921.000564:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.064490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204921.064493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204921.064501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204921.064502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204921.064504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.064505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204921.064514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204921.064514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204921.064521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204921.064521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204921.064527:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204921.064527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204921.064533:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204921.064536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204921.064540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204921.064541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.064545:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204921.064549:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204921.256461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204921.256471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204921.256479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204921.256480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204921.256483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204921.256484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204921.256488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204921.256489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204921.256571:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204921.256576:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204921.256585:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204921.576547:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204921.576559:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204921.576561:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204921.576567:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204921.576571:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204921.576575:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204921.576578:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204921.576580:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204921.576583:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.024485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204922.024492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204922.024494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204922.024500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204922.024503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204922.024508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204922.024510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204922.024512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204922.024514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.088524:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204922.088525:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204922.088541:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204922.088542:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204922.088547:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204922.088548:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204922.088558:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204922.088558:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204922.088567:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204922.088567:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204922.088572:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204922.088575:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204922.088577:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204922.088580:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.088586:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204922.088591:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204922.088593:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204922.088597:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204922.280549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204922.280557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204922.280565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204922.280566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204922.280570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204922.280571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204922.280575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204922.280576:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204922.280602:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204922.280605:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204922.280608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204922.600524:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204922.600531:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204922.600534:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204922.600538:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204922.600541:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204922.600545:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204922.600548:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204922.600551:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204922.600554:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.048464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204923.048472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204923.048474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204923.048777:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204923.048782:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204923.048785:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204923.048786:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204923.048788:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204923.048790:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.112460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204923.112460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204923.112467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204923.112467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204923.112471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204923.112471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204923.112479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204923.112479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204923.112485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204923.112486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204923.112492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.112492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.112499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204923.112502:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.112504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.112506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204923.112509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204923.112512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204923.304465:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204923.304467:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204923.304470:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304474:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204923.304475:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304477:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304479:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204923.304484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204923.304488:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204923.304489:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204923.304491:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304492:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204923.304494:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304495:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304497:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204923.304502:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204923.304503:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204923.304504:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304506:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204923.304507:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304508:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204923.304510:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204923.304565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204923.304568:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204923.304571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204923.624471:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301050369, at 4301050880 00000100:00000040:0.0:1788204923.624476:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301050369, at 4301050880 00000100:00000010:1.0:1788204923.624487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1b40. 00000100:00000010:0.0:1788204923.624489:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1840. 00000100:00000001:3.0:1788204923.624508:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204923.624513:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204923.624515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204923.624519:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204923.624523:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204923.624526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.624528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204923.624531:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.624533:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204924.072483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204924.072491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204924.072493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204924.072498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204924.072502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204924.072505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.072508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204924.072510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.072513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204924.136482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204924.136482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204924.136490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204924.136490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204924.136492:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204924.136494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.136495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.136501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204924.136502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:2.0:1788204924.136502:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204924.136505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204924.136506:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204924.136508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204924.136509:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204924.136509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.136512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204924.136513:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204924.136513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204924.136517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204924.136521:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6378 00000100:00000001:3.0:1788204924.136521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204924.136524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204924.136527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.136531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204924.136531:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204924.136540:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204924.136543:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204924.136546:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136550:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.136552:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136555:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204924.136558:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204924.136560:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204924.136563:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204924.136566:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204924.136569:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204924.136572:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204924.136575:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136576:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.136578:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136580:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204924.136583:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204924.136586:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136588:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204924.136591:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204924.136593:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204924.136596:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.136598:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204924.136601:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204924.136610:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204924.136613:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204924.136617:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204924.136618:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.136621:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204924.136624:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204924.136628:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204924.136634:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204924.136639:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204924.136646:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204924.136653:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204924.136656:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204924.136659:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204924.136679:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204924.136682:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204924.136685:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204924.136686:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204924.136688:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204924.136691:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204924.136693:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6378 00000400:00000200:2.0:1788204924.136698:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204924.136702:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204924.136703:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788204924.136705:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204924.136707:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788204924.136710:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204924.136710:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204924.136714:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204924.136717:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204924.136718:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136722:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204924.136725:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204924.136728:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204924.136730:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204924.136732:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788204924.136734:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204924.136735:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204924.136737:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204924.136739:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788204924.136740:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204924.136742:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204924.136744:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204924.136745:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.136747:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.136751:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00100000:3.0:1788204924.136751:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205911168:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204924.136753:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204924.136757:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1788204924.136758:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204924.136761:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204924.136762:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788204924.136764:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.136765:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204924.136767:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204924.136768:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204924.136773:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1788204924.136774:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204924.136779:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.136779:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.136785:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204924.136789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788204924.136791:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788204924.136793:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000008:00000001:2.0:1788204924.136796:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204924.136799:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:2.0:1788204924.136800:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204924.136806:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911168, portal 25 00000100:00000010:2.0:1788204924.136807:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204924.136811:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1788204924.136813:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:3.0:1788204924.136814:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 02000000:00000001:2.0:1788204924.136818:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204924.136820:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.136822:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204924.136824:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204924.136826:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.136826:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204924.136831:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204924.136835:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204924.136855:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204924.136862:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204924.136864:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204924.136868:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.136871:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204924.136878:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204924.136882:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204924.136887:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205911168, offset 0 00000100:00000001:2.0:1788204924.136894:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.136895:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204924.136899:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204924.136903:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204924.136906:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204924.136906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788204924.136909:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204924.136912:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204924.136914:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:2.0:1788204924.136917:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6378 00000400:00000200:3.0:1788204924.136922:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204924.136926:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204924.136930:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204924.136932:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204924.136936:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58974:58974:256:4294967295] 192.168.201.151@tcp LPNI seq info [58954:58954:8:4294967295] 00000400:00000200:3.0:1788204924.136941:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204924.136946:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204924.136952:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204924.136955:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204924.136961:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204924.136961:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788204924.136965:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.136968:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136970:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204924.136972:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204924.136974:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000800:00000200:3.0:1788204924.136975:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788204924.136976:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204924.136979:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204924.136981:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:3.0:1788204924.136982:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000020:00000001:2.0:1788204924.136983:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204924.136988:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136990:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204924.136991:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:2.0:1788204924.136993:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.136996:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788204924.136996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000020:00000040:2.0:1788204924.136998:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204924.137003:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.1:1788204924.137006:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204924.137007:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204924.137012:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788204924.137012:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 02000000:00000001:2.0:1788204924.137015:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:3.0:1788204924.137016:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788204924.137020:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204924.137020:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000100:00000001:2.0:1788204924.137022:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204924.137025:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204924.137026:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:2.0:1788204924.137027:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.137030:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.137033:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1788204924.137035:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204924.137038:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788204924.137038:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 02000000:00000001:2.0:1788204924.137039:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788204924.137041:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 02000000:00000001:2.0:1788204924.137042:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.137046:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204924.137048:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788204924.137049:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204924.137053:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204924.137053:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204924.137059:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:3.0:1788204924.137063:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204924.137065:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137067:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204924.137074:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204924.137075:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137078:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204924.137084:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.137085:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204924.137090:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.137090:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204924.137096:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301055489 00000100:00000040:2.0:1788204924.137098:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204924.137101:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4300. 00000100:00000001:3.0:1788204924.137109:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204924.137110:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204924.137115:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204924.137116:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301055489 00000100:00080000:2.0:1788204924.137119:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000010:3.0:1788204924.137120:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4480. 00000100:00000001:2.0:1788204924.137122:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204924.137124:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788204924.137125:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204924.137127:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204924.137130:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204924.137131:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204924.137135:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6378 00000100:00000001:3.0:1788204924.137136:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1788204924.137140:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788204924.137145:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204924] req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204924.137147:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204924.137151:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204924.137157:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.137162:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.137165:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000001:3.0:1788204924.137166:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788204924.137167:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000001:3.0:1788204924.137170:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788204924.137171:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204924.137175:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000400:00000010:3.0:1788204924.137175:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000020:00000001:2.0:1788204924.137177:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204924.137181:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:3.0:1788204924.137181:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204924.137185:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000020:00000001:2.0:1788204924.137186:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204924.137188:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1788204924.137190:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000200:3.0:1788204924.137195:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204924.137196:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.137201:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204924.137205:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204924.137209:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204924.137213:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204924.137220:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000200:3.0:1788204924.137221:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204924.137222:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204924.137227:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204924.137231:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204924.137233:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788204924.137236:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137236:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204924.137241:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204924.137242:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137245:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788204924.137246:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1788204924.137250:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788204924.137251:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204924.137257:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137259:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1788204924.137264:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000040:3.0:1788204924.137264:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204924.137270:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204924.137275:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204924.137276:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204924.137278:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204924.137280:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137280:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137282:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204924.137285:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204924.137286:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788204924.137289:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204924.137289:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204924.137293:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204924.137293:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204924.137299:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 02000000:00000010:2.0:1788204924.137301:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:3.0:1788204924.137301:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788204924.137306:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137306:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204924.137315:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204924.137318:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205911168:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204924.137326:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204924.137330:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204924.137331:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204924.137334:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137336:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205911168/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204924.137340:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137352:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788204924.137356:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137356:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204924.137361:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 00000100:00000040:2.0:1788204924.137362:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6389) 02000000:00000001:3.0:1788204924.137381:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204924.137383:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204924.137385:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204924.137386:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204924.137388:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:3.0:1788204924.137391:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204924.137392:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204924.137394:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788204924.137398:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204924.137399:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137402:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137409:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204924.137411:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137413:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204924.137415:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204924.137416:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204924.137420:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137428:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204924.137429:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137431:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204924.137433:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204924.137435:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.137438:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911296:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204924.137441:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204924.137443:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204924.137445:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137446:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204924.137448:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788204924.137449:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.137452:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204924.137454:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204924.137456:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204924.137458:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911296, portal 10 00000100:00000001:3.0:1788204924.137460:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204924.137461:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788204924.137466:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137472:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204924.137474:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204924.137476:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204924.137478:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205911296, offset 0 00000400:00000200:3.0:1788204924.137482:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137486:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137489:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204924.137492:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204924.137495:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204924.137497:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204924.137500:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137502:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58975:58975:256:4294967295] 192.168.201.151@tcp LPNI seq info [58955:58955:8:4294967295] 00000400:00000200:3.0:1788204924.137506:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137512:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204924.137515:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137518:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204924.137520:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204924.137522:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204924.137525:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204924.137526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1788204924.137528:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137530:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204924.137532:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204924.137534:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204924.137536:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204924.137537:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204924.137538:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204924.137541:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137549:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.137551:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137555:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137556:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204924.137558:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137560:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204924.137562:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204924.137564:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301051547 00000100:00000001:3.0:1788204924.137565:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137567:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301051547 00000100:00000001:3.0:1788204924.137568:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137569:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204924.137571:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137572:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204924.137574:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204924] req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204924.137578:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204924.137579:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204924.137580:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204924.137581:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204924.137582:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204924.137584:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204924.137585:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204924.137590:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137593:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204924.137594:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137595:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137596:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204924.137598:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137602:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204924.137604:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137607:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204924.137609:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137609:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137610:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137611:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204924.137612:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204924.137614:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204924.137616:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204924.137617:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204924.137619:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204924.137623:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911296:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204924.137625:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204924.137627:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137631:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204924.137631:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137633:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204924.137634:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204924.137635:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.137637:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911424:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204924.137639:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204924.137640:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204924.137641:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137642:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204924.137644:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788204924.137645:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.137647:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204924.137648:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204924.137650:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204924.137652:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911424, portal 4 00000100:00000001:3.0:1788204924.137653:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204924.137654:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788204924.137657:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137661:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204924.137662:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204924.137663:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204924.137665:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205911424, offset 0 00000400:00000200:3.0:1788204924.137667:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137669:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137671:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204924.137673:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204924.137674:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204924.137676:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204924.137677:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137679:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58976:58976:256:4294967295] 192.168.201.151@tcp LPNI seq info [58956:58956:8:4294967295] 00000400:00000200:3.0:1788204924.137681:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137685:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204924.137688:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137689:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204924.137692:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204924.137693:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204924.137696:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204924.137697:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1788204924.137699:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137701:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204924.137702:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204924.137704:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204924.137705:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204924.137706:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204924.137708:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204924.137709:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137714:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.137715:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137719:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137720:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204924.137721:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137723:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204924.137725:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204924.137726:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301051433 00000100:00000001:3.0:1788204924.137727:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137728:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301051433 00000100:00000001:3.0:1788204924.137730:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137731:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204924.137732:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137733:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204924.137735:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204924] req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204924.137739:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204924.137740:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204924.137741:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204924.137742:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204924.137743:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204924.137744:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204924.137746:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204924.137750:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137753:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204924.137754:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137755:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137756:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204924.137757:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137761:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204924.137763:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137766:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204924.137767:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137768:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137769:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137770:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204924.137771:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204924.137772:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204924.137774:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204924.137775:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204924.137778:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204924.137786:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911424:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204924.137788:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204924.137790:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137793:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204924.137794:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137795:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204924.137796:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204924.137797:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.137798:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911552:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204924.137800:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204924.137800:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204924.137801:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204924.137802:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204924.137804:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788204924.137805:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204924.137807:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204924.137808:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204924.137809:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204924.137811:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911552, portal 4 00000100:00000001:3.0:1788204924.137813:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204924.137814:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204924.137819:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137825:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204924.137827:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204924.137829:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204924.137831:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205911552, offset 0 00000400:00000200:3.0:1788204924.137835:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137849:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137853:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204924.137856:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204924.137858:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204924.137860:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204924.137863:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204924.137865:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58977:58977:256:4294967295] 192.168.201.151@tcp LPNI seq info [58957:58957:8:4294967295] 00000400:00000200:3.0:1788204924.137869:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204924.137874:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204924.137877:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137879:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204924.137881:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204924.137883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204924.137886:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204924.137887:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1788204924.137890:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204924.137891:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204924.137893:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204924.137895:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204924.137897:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204924.137898:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204924.137899:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204924.137902:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137906:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.137908:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204949 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137912:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137913:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204924.137914:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204924.137916:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204924.137918:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204924.137920:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301051439 00000100:00000001:3.0:1788204924.137921:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301051439 00000100:00000001:3.0:1788204924.137924:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301055489 00000100:00000001:3.0:1788204924.137926:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204924.137928:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.137930:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204924.137933:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204924] req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204924.137940:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204924.137941:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204924.137943:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204924.137945:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204924.137946:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204924.137948:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204924.137951:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204924.137959:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137966:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204924.137967:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137969:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137970:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204924.137972:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137977:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204924.137979:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.137982:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204924.137984:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137985:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204924.137986:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.137987:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204924.137988:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204924.137990:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204924.137991:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204924.137992:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204924.137994:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204924.137998:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911552:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204924.138000:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.138001:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.138003:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205911296/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.138007:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204924.138008:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204924.138009:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788204924.138011:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204924.138011:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204924.138012:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204924.138014:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204924.138015:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788204924.138017:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204924.138017:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204924.138018:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204924.138020:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204924.138020:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.138021:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.138023:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205911424/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.138030:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204924.138031:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204924.138032:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788204924.138033:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204924.138034:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204924.138034:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204924.138035:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204924.138036:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:3.0:1788204924.138037:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204924.138038:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204924.138038:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788204924.138039:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204924.138040:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.138041:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204924.138042:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205911552/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204924.138045:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204924.138046:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204924.138047:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788204924.138048:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204924.138048:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204924.138049:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204924.138050:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204924.138050:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:3.0:1788204924.138051:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204924.138052:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204924.138053:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204924.138054:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204924.138054:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.138055:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.138057:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204924.138058:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.138059:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204924.138060:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204924.328521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204924.328527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204924.328533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204924.328534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204924.328538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204924.328539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204924.328542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204924.328543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204924.328558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204924.328560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204924.328563:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204924.648427:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204924.648434:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204924.648436:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.648448:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204924.648451:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204924.648455:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.648458:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204924.648460:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.648463:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.096449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204925.096461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204925.096463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.096467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204925.096471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204925.096474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.096475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.096477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.096479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.160450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204925.160456:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204925.160458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204925.160461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.160462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204925.160465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.160467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204925.160469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204925.160470:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204925.160475:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.160475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204925.160479:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204925.160480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.160483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.160485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.160486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204925.160490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.160493:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204925.352474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204925.352480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204925.352485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204925.352487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204925.352489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204925.352491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204925.352494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204925.352495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204925.352549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204925.352553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204925.352556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204925.672485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204925.672490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204925.672491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.672495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204925.672497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204925.672500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.672502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204925.672505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.672507:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.120469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204926.120474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204926.120475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204926.120479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204926.120481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204926.120484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204926.120486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204926.120488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204926.120490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.184481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204926.184481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204926.184488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204926.184489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204926.184491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.184492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204926.184497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204926.184498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204926.184502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204926.184504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204926.184505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204926.184508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.184509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204926.184512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.184515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.184516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204926.184519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204926.184522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204926.376528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204926.376535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204926.376541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204926.376543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204926.376546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204926.376547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204926.376550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204926.376551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204926.376569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204926.376571:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204926.376574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204926.696433:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204926.696439:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204926.696441:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204926.696446:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204926.696448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204926.696451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.696453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204926.696454:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.696456:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.144434:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204927.144441:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204927.144443:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204927.144448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204927.144451:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204927.144454:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204927.144457:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204927.144459:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204927.144462:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.208448:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204927.208448:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204927.208455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204927.208458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204927.208463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204927.208465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204927.208467:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204927.208469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.208472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204927.208479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204927.208480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204927.208485:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204927.208486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204927.208490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204927.208491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204927.208494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204927.208497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204927.208500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204927.400469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204927.400475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204927.400480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204927.400482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204927.400485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204927.400486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204927.400489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204927.400490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204927.400545:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204927.400548:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204927.400551:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204927.720486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204927.720493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204927.720495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204927.720500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204927.720503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204927.720508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204927.720511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204927.720514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204927.720516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204928.168465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204928.168470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204928.168471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204928.168475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204928.168478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204928.168481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.168482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204928.168484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.168486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204928.232462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204928.232462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204928.232469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204928.232469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204928.232472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204928.232472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204928.232477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204928.232478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204928.232482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204928.232483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204928.232487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204928.232487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.232492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204928.232494:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.232496:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204928.232497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204928.232504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204928.232506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204928.424513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204928.424518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204928.424521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204928.424522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204928.424524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204928.424525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204928.424526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204928.424527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204928.424541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204928.424542:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204928.424545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204928.744439:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204928.744440:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301055489, at 4301056000 00000100:00000001:3.0:1788204928.744445:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204928.744447:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204928.744451:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301055489, at 4301056000 00000100:00100000:3.0:1788204928.744451:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204928.744452:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000010:0.0:1788204928.744462:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000001:3.0:1788204928.744463:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204928.744467:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204928.744469:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204928.744473:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204928.744476:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204929.192523:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204929.192531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204929.192534:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204929.192538:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204929.192542:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204929.192546:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204929.192547:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204929.192549:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204929.192552:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204929.256487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204929.256488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204929.256496:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204929.256502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204929.256503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788204929.256504:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204929.256513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204929.256514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.256515:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204929.256522:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204929.256529:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1788204929.256529:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204929.256529:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204929.256538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204929.256539:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6384 00000100:00000001:3.0:1788204929.256540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204929.256543:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204929.256548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204929.256549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204929.256551:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204929.256557:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204929.256559:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.256560:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788204929.256566:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204929.256567:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204929.256572:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204929.256575:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204929.256576:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256584:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.256588:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256591:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204929.256594:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204929.256595:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204929.256597:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204929.256600:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204929.256602:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204929.256605:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204929.256608:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256611:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.256613:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256616:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204929.256619:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204929.256623:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256626:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204929.256628:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204929.256632:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204929.256636:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.256638:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204929.256641:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204929.256652:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204929.256656:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204929.256660:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204929.256662:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.256664:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204929.256667:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204929.256671:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204929.256680:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204929.256686:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204929.256695:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204929.256703:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204929.256706:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204929.256710:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204929.256732:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204929.256735:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204929.256738:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204929.256739:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204929.256740:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204929.256742:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204929.256744:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6384 00000400:00000200:0.0:1788204929.256761:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204929.256765:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204929.256768:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204929.256771:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256779:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.256783:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256786:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204929.256788:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204929.256790:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204929.256791:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204929.256792:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204929.256797:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204929.256798:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204929.256804:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204929.256804:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204929.256810:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204929.256817:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204929.256819:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204929.256822:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.256828:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.256836:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204929.256844:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204929.256849:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788204929.256852:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204929.256853:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.256861:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.256861:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204929.256867:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204929.256868:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204929.256872:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204929.256875:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788204929.256878:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205911680:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788204929.256881:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.256886:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788204929.256887:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204929.256893:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788204929.256894:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204929.256897:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788204929.256901:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.256903:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204929.256905:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204929.256913:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 02000000:00000010:3.0:1788204929.256914:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 00000020:00000040:0.0:1788204929.256918:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204929.256919:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204929.256922:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204929.256924:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.256925:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204929.256928:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204929.256931:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.256932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:0.0:1788204929.256936:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000400:00000010:3.0:1788204929.256939:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204929.256941:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204929.256949:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204929.256950:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:0.0:1788204929.256961:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204929.256962:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911680, portal 25 00000100:00000001:0.0:1788204929.256964:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.256970:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1788204929.256973:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.256975:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788204929.256990:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204929.256992:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204929.256994:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204929.257009:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204929.257009:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204929.257013:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:3.0:1788204929.257014:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204929.257016:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788204929.257019:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000001:0.0:1788204929.257021:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204929.257026:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6384 00000100:00000200:3.0:1788204929.257027:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205911680, offset 0 00000400:00000200:0.0:1788204929.257036:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204929.257042:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204929.257050:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204929.257050:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204929.257060:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257062:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.257064:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257072:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204929.257073:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788204929.257082:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204929.257084:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204929.257085:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204929.257088:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204929.257089:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:0.0:1788204929.257091:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204929.257095:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204929.257098:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1788204929.257099:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257104:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204929.257107:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:0.0:1788204929.257109:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257115:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204929.257116:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204929.257123:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:0.0:1788204929.257125:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204929.257133:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58978:58978:256:4294967295] 192.168.201.151@tcp LPNI seq info [58958:58958:8:4294967295] 00000100:00080000:0.0:1788204929.257135:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257144:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.257144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204929.257155:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204929.257160:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204929.257170:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.257171:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788204929.257177:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204929.257182:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:3.0:1788204929.257183:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000008:00000001:0.0:1788204929.257192:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204929.257194:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000001:0.0:1788204929.257196:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204929.257203:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000800:00000200:3.0:1788204929.257204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788204929.257210:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.0:1788204929.257212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 02000000:00000001:0.0:1788204929.257217:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204929.257221:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204929.257227:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204929.257228:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204929.257231:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204929.257235:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204929.257235:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 02000000:00000010:0.0:1788204929.257249:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000800:00000100:3.0:1788204929.257249:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.257259:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000100:00000001:0.0:1788204929.257261:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.257267:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:0.0:1788204929.257273:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204929.257278:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204929.257288:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000100:00000001:0.0:1788204929.257293:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1788204929.257294:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:0.0:1788204929.257298:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.257301:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1788204929.257307:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204929.257309:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257339:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204929.257344:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204929.257349:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788204929.257349:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204929.257358:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204929.257363:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204929.257392:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.257395:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204929.257404:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204929.257410:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204929.257413:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204929.257414:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204929.257420:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301060610 00000100:00080000:0.0:1788204929.257421:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6384 00000400:00000200:0.0:1788204929.257431:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:3.0:1788204929.257431:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4d20. 00000400:00000200:0.0:1788204929.257438:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204929.257441:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204929.257442:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788204929.257446:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257450:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204929.257455:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257457:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204929.257457:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301060610 00000020:00000001:0.0:1788204929.257462:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204929.257465:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000010:3.0:1788204929.257466:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4f60. 00000020:00000001:0.0:1788204929.257471:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204929.257475:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204929.257478:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204929.257478:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788204929.257486:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204929.257490:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788204929.257491:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257499:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204929.257502:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204929.257503:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204929.257510:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204929.257511:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:0.0:1788204929.257513:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204929.257518:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00100000:3.0:1788204929.257520:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204929.257522:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204929.257527:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204929.257533:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204929.257539:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.257547:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204929.257551:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000001:3.0:1788204929.257552:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:0.0:1788204929.257559:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204929.257559:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788204929.257561:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204929.257563:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000100:00000010:0.0:1788204929.257566:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000400:00000200:3.0:1788204929.257569:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000020:00000040:0.0:1788204929.257570:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204929.257577:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788204929.257577:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 02000000:00000001:0.0:1788204929.257584:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204929.257588:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204929.257589:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1788204929.257595:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204929.257600:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204929.257601:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204929.257604:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204929.257611:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204929.257620:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204929.257622:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204929.257655:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204929.257660:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.257667:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788204929.257674:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257674:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257683:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.257688:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204929.257697:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204929.257703:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6394) 00000100:00100000:3.0:1788204929.257705:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257715:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204929.257720:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257727:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204929.257730:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257732:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.257733:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257735:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204929.257737:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204929.257741:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204929.257745:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204929.257759:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204929.257764:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204929.257772:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205911680:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204929.257776:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257778:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.257781:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205911680/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257787:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204929.257790:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204929.257793:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788204929.257795:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204929.257798:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204929.257800:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204929.257802:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204929.257804:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204929.257807:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204929.257809:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204929.257811:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204929.257814:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204929.257816:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257818:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.257824:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204929.257825:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.257828:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204929.257829:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204929.257830:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204929.257834:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257841:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204929.257843:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.257845:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204929.257847:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204929.257849:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204929.257852:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911808:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204929.257855:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204929.257857:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204929.257859:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.257860:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204929.257863:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788204929.257865:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.257869:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204929.257872:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204929.257875:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204929.257877:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911808, portal 10 00000100:00000001:3.0:1788204929.257879:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204929.257880:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204929.257886:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257893:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204929.257895:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204929.257897:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204929.257899:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205911808, offset 0 00000400:00000200:3.0:1788204929.257903:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204929.257906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.257909:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204929.257912:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204929.257915:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204929.257917:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204929.257919:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204929.257922:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58979:58979:256:4294967295] 192.168.201.151@tcp LPNI seq info [58959:58959:8:4294967295] 00000400:00000200:3.0:1788204929.257926:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.257933:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204929.257937:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.257939:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204929.257943:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204929.257946:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204929.257951:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204929.257954:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788204929.257958:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.257959:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204929.257962:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204929.257965:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204929.257968:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204929.257969:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204929.257972:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204929.257975:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257982:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.257985:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.257991:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.257993:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204929.257995:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204929.257999:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204929.258002:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204929.258004:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301056667 00000100:00000001:3.0:1788204929.258006:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258008:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301056667 00000100:00000001:3.0:1788204929.258010:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258012:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204929.258014:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258015:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204929.258018:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204929.258026:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204929.258027:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204929.258029:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204929.258030:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204929.258032:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204929.258033:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204929.258036:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204929.258045:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258050:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204929.258051:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258053:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258054:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204929.258058:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258064:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204929.258067:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258074:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204929.258077:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258078:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258079:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258081:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204929.258083:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204929.258085:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204929.258088:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204929.258090:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204929.258093:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204929.258100:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911808:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204929.258104:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204929.258107:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258115:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204929.258116:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.258118:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204929.258119:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204929.258121:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204929.258124:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911936:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204929.258127:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204929.258129:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204929.258130:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.258132:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204929.258134:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788204929.258136:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.258139:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204929.258142:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204929.258145:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204929.258147:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205911936, portal 4 00000100:00000001:3.0:1788204929.258148:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204929.258150:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204929.258155:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258161:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204929.258163:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204929.258165:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204929.258167:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205911936, offset 0 00000400:00000200:3.0:1788204929.258171:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204929.258174:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.258177:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204929.258179:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204929.258182:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204929.258184:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204929.258186:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204929.258189:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58980:58980:256:4294967295] 192.168.201.151@tcp LPNI seq info [58960:58960:8:4294967295] 00000400:00000200:3.0:1788204929.258193:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.258199:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204929.258203:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.258206:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204929.258209:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204929.258212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204929.258216:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204929.258218:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788204929.258222:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.258224:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204929.258226:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204929.258230:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204929.258232:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204929.258234:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204929.258236:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204929.258239:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258247:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.258249:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258255:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258257:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204929.258259:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204929.258262:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204929.258265:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204929.258267:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301056553 00000100:00000001:3.0:1788204929.258269:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258271:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301056553 00000100:00000001:3.0:1788204929.258273:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258275:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204929.258277:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258279:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204929.258282:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204929.258289:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204929.258290:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204929.258292:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204929.258294:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204929.258295:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204929.258297:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204929.258300:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204929.258308:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258320:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204929.258321:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258323:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258324:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204929.258327:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258333:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204929.258336:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258344:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204929.258346:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258347:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258348:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258350:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204929.258352:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204929.258354:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204929.258357:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204929.258359:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204929.258383:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204929.258391:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205911936:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204929.258395:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204929.258399:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258405:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204929.258406:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.258409:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204929.258411:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204929.258413:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204929.258415:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912064:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204929.258418:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204929.258420:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204929.258421:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204929.258423:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204929.258425:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788204929.258427:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204929.258430:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204929.258433:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204929.258435:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204929.258438:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912064, portal 4 00000100:00000001:3.0:1788204929.258440:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204929.258441:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204929.258446:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258452:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204929.258454:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204929.258456:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204929.258457:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205912064, offset 0 00000400:00000200:3.0:1788204929.258461:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204929.258464:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.258468:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204929.258470:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204929.258473:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204929.258475:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204929.258477:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204929.258480:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58981:58981:256:4294967295] 192.168.201.151@tcp LPNI seq info [58961:58961:8:4294967295] 00000400:00000200:3.0:1788204929.258483:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204929.258489:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204929.258494:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.258497:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204929.258500:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204929.258502:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204929.258507:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204929.258509:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788204929.258513:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204929.258514:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204929.258517:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204929.258520:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204929.258522:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204929.258524:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204929.258526:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204929.258529:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258536:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.258539:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258545:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258547:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204929.258550:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204929.258552:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204929.258555:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204929.258557:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301056560 00000100:00000001:3.0:1788204929.258559:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258561:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301056560 00000100:00000001:3.0:1788204929.258563:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301060610 00000100:00000001:3.0:1788204929.258565:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204929.258567:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258568:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204929.258572:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204929.258579:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204929.258580:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204929.258582:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204929.258584:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204929.258585:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204929.258587:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204929.258590:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204929.258599:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258605:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204929.258606:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258608:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258609:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204929.258611:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258617:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204929.258621:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258627:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204929.258629:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258631:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204929.258632:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258634:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204929.258636:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204929.258638:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204929.258641:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204929.258642:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204929.258646:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204929.258654:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912064:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204929.258657:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258659:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.258662:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205911808/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258668:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204929.258669:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204929.258671:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788204929.258673:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204929.258674:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204929.258675:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204929.258677:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204929.258679:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788204929.258682:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204929.258683:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204929.258684:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204929.258687:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204929.258688:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258689:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.258693:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205911936/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258698:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204929.258699:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204929.258701:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788204929.258703:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204929.258704:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204929.258705:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204929.258707:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204929.258708:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788204929.258710:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204929.258711:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204929.258712:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204929.258714:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204929.258715:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258717:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204929.258720:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205912064/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204929.258726:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204929.258727:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204929.258728:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788204929.258730:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204929.258731:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204929.258732:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204929.258734:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204929.258735:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788204929.258737:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204929.258738:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204929.258740:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204929.258741:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204929.258742:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258744:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.258746:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204929.258759:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.258762:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204929.258763:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204929.448522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204929.448529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204929.448533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204929.448535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204929.448537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204929.448538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204929.448540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204929.448542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204929.448589:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204929.448592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204929.448595:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204929.768497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204929.768505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204929.768507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204929.768514:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204929.768517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204929.768521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.768523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204929.768526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.768528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.216497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204930.216507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204930.216509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204930.216516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204930.216519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204930.216523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204930.216525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204930.216527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204930.216530:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.280480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204930.280480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204930.280489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204930.280490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204930.280492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.280493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.280499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204930.280500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204930.280504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204930.280506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204930.280508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204930.280511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.280511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204930.280514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.280517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.280518:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204930.280521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204930.280525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204930.472541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204930.472548:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204930.472579:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204930.472589:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472598:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204930.472600:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472614:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472616:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472619:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204930.472620:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204930.472623:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204930.472624:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204930.472699:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204930.472706:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204930.472708:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204930.792515:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204930.792523:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204930.792525:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204930.792530:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204930.792533:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204930.792537:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204930.792539:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204930.792541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204930.792544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.240498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204931.240509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204931.240512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204931.240519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204931.240523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204931.240528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.240530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204931.240532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.240535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204931.304520:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204931.304529:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204931.304531:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204931.304536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204931.304539:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204931.304543:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204931.304545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204931.304547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204931.304550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.304644:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204931.304647:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204931.304649:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204931.304651:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204931.304653:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204931.304654:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.304656:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204931.304657:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.304659:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204931.496445:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204931.496454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204931.496458:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204931.496459:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204931.496461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204931.496462:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204931.496465:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204931.496466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204931.496533:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204931.496538:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204931.496542:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204931.816497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204931.816505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204931.816507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204931.816512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204931.816515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204931.816519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204931.816522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204931.816524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204931.816527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204932.264501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204932.264511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204932.264513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204932.264521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204932.264525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204932.264529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204932.264533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204932.264536:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204932.264539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204932.328510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204932.328510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204932.328520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204932.328520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204932.328522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.328524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204932.328528:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204932.328531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204932.328533:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204932.328540:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204932.328540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204932.328548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204932.328548:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204932.328551:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204932.328554:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.328555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204932.328561:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204932.328564:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204932.520556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204932.520563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204932.520569:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204932.520571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204932.520574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204932.520576:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204932.520580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204932.520582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204932.520605:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204932.520608:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204932.520611:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204932.840497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204932.840504:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204932.840513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204932.840517:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204932.840521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204932.840525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204932.840528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204932.840530:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204932.840533:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204933.288480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204933.288488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204933.288490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204933.288495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204933.288498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204933.288502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.288504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204933.288507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.288509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204933.352489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204933.352489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204933.352505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204933.352506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204933.352513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204933.352514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204933.352529:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204933.352530:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204933.352540:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204933.352541:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204933.352543:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.352548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204933.352548:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.352552:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.352554:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204933.352562:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204933.352564:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204933.352568:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204933.544520:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204933.544522:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204933.544524:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544529:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204933.544530:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544532:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544535:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204933.544538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204933.544545:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204933.544547:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204933.544548:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544549:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204933.544550:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544551:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544553:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204933.544556:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204933.544557:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204933.544558:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544558:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204933.544559:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544560:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204933.544561:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204933.544620:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204933.544624:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204933.544627:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204933.864506:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301060610, at 4301061120 00000100:00000040:0.0:1788204933.864518:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301060610, at 4301061120 00000100:00000010:1.0:1788204933.864531:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000010:0.0:1788204933.864533:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000001:3.0:1788204933.864565:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204933.864575:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204933.864578:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204933.864586:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204933.864591:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204933.864596:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204933.864600:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204933.864603:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204933.864607:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.312457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204934.312463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204934.312465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204934.312469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204934.312472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204934.312476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204934.312478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204934.312480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204934.312483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.376480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204934.376486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204934.376488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204934.376496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204934.376497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204934.376498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.376505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204934.376507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204934.376508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.376514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204934.376515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204934.376519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204934.376520:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204934.376536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376537:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204934.376539:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.376542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204934.376561:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204934.376565:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204934.376568:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204934.376571:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204934.376574:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204934.376581:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6389 00000400:00000200:3.0:1788204934.376589:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204934.376594:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204934.376597:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204934.376599:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376602:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.376604:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376605:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204934.376608:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204934.376610:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204934.376612:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204934.376614:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204934.376616:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204934.376618:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204934.376620:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376621:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.376623:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376624:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204934.376627:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204934.376629:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376631:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204934.376633:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204934.376636:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204934.376642:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376643:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204934.376645:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204934.376655:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0000. 00000020:00000040:3.0:1788204934.376657:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204934.376688:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204934.376690:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376694:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204934.376697:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204934.376702:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204934.376710:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21000. 00000100:00000001:3.0:1788204934.376717:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204934.376726:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204934.376734:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204934.376738:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204934.376742:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee757f9f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204934.376765:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204934.376767:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204934.376769:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204934.376771:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204934.376772:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204934.376774:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204934.376776:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6389 00000400:00000200:3.0:1788204934.376778:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204934.376780:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204934.376782:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204934.376784:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376785:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.376787:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376788:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204934.376790:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204934.376791:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788204934.376792:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204934.376794:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788204934.376795:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788204934.376796:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204934.376799:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376801:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.376802:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204934.376804:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204934.376804:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204934.376808:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788204934.376809:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788204934.376811:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204934.376812:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788204934.376813:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204934.376816:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1788204934.376820:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204934.376820:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204934.376823:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376824:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204934.376828:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204934.376833:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.376837:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204934.376837:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.376840:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204934.376840:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1880. 00000020:00000040:3.0:1788204934.376843:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204934.376844:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204934.376845:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204934.376848:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204934.376848:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376853:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204934.376854:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204934.376855:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204934.376857:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204934.376858:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205912192:192.168.201.151@tcp:250:kworker.0 02000000:00000010:3.0:1788204934.376861:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22c00. 00000100:00000001:2.0:1788204934.376863:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788204934.376864:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.376867:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204934.376869:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204934.376870:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788204934.376872:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788204934.376877:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1788204934.376878:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:3.0:1788204934.376880:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.376883:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204934.376884:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072ac1564] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204934.376891:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.376893:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:3.0:1788204934.376894:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204934.376897:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204934.376898:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204934.376899:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204934.376901:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204934.376903:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6389 00000400:00000200:3.0:1788204934.376906:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204934.376907:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204934.376909:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1788204934.376910:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000040:3.0:1788204934.376911:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376915:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.376917:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376918:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204934.376919:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204934.376920:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000010:2.0:1788204934.376921:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000020:00000001:3.0:1788204934.376921:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204934.376924:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788204934.376926:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788204934.376928:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:2.0:1788204934.376929:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912192, portal 25 00000100:00000040:3.0:1788204934.376931:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204934.376933:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204934.376934:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204934.376936:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:3.0:1788204934.376937:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376941:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204934.376942:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000040:2.0:1788204934.376945:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204934.376946:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376950:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204934.376953:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204934.376956:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204934.376957:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788204934.376959:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204934.376962:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204934.376962:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:2.0:1788204934.376965:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000008:00100000:3.0:1788204934.376965:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788204934.376969:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204934.376970:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205912192, offset 0 00000100:00000001:3.0:1788204934.376971:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204934.376976:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1180. 00000020:00000040:3.0:1788204934.376978:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:2.0:1788204934.376980:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788204934.376981:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204934.376983:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.376984:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204934.376985:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204934.376986:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204934.376989:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21c00. 00000100:00000001:3.0:1788204934.376991:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.376995:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788204934.376995:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788204934.377003:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:3.0:1788204934.377005:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204934.377009:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.377010:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204934.377017:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:3.0:1788204934.377017:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003da94af2] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204934.377022:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204934.377026:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204934.377032:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58982:58982:256:4294967295] 192.168.201.151@tcp LPNI seq info [58962:58962:8:4294967295] 00000100:00000001:3.0:1788204934.377034:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.377038:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:3.0:1788204934.377041:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204934.377048:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1788204934.377049:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:3.0:1788204934.377053:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204934.377056:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.0:1788204934.377057:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204934.377061:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204934.377065:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6389 00000800:00000010:2.0:1788204934.377066:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000400:00000200:3.0:1788204934.377070:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204934.377074:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204934.377077:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1788204934.377078:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:3.0:1788204934.377082:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204934.377083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1788204934.377086:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.377090:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.1:1788204934.377092:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204934.377093:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:2.1:1788204934.377098:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1788204934.377101:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.377103:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000020:00000001:3.0:1788204934.377104:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204934.377106:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788204934.377107:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788204934.377108:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1788204934.377110:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788204934.377114:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204934.377115:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1788204934.377119:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:2.0:1788204934.377123:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00000040:3.0:1788204934.377123:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:2.0:1788204934.377127:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204934.377127:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204934.377130:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204934.377132:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204934.377134:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1788204934.377134:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788204934.377139:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000200:2.0:1788204934.377141:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204934.377141:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204934.377144:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204934.377147:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204934.377151:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.377154:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204934.377157:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788204934.377159:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377160:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204934.377161:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204934.377165:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1f80. 00000100:00000040:2.0:1788204934.377168:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204934.377169:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788204934.377172:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204934.377174:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.377177:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204934.377179:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204934.377182:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377185:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204934.377187:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23800. 00000100:00000001:2.0:1788204934.377188:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204934.377194:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377196:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.377201:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788204934.377201:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204934.377205:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301065729 00000100:00000010:2.0:1788204934.377212:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ba0. 00000100:00000001:3.0:1788204934.377214:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204934.377218:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377219:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1788204934.377224:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002399dd83] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377229:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301065729 00000100:00000010:2.0:1788204934.377235:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1300. 00000100:00000001:3.0:1788204934.377236:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204934.377242:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6399) 00000100:00000001:2.0:1788204934.377250:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204934.377257:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204934.377260:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377263:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204934.377268:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204934/real 1788204934] req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204934.377275:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204934.377276:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204934.377278:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204934.377280:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204934.377281:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204934.377284:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204934.377287:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204934.377295:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377302:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204934.377304:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377306:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377308:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204934.377312:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377321:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204934.377325:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377330:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204934.377333:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377334:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377335:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377337:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204934.377338:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204934.377340:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204934.377342:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204934.377344:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204934.377347:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204934.377352:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205912192:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204934.377356:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377357:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.377359:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205912192/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377390:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204934.377393:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204934.377394:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204934.377397:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204934.377398:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204934.377400:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204934.377401:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204934.377403:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204934.377406:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204934.377408:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204934.377410:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:2.0:1788204934.377412:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204934.377413:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377414:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377422:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204934.377423:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377424:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204934.377426:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204934.377427:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204934.377430:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377438:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204934.377439:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.377441:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204934.377443:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204934.377450:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204934.377454:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912320:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204934.377457:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204934.377459:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204934.377460:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.377462:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204934.377465:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204934.377467:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.377471:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204934.377473:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204934.377476:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204934.377477:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912320, portal 10 00000100:00000001:2.0:1788204934.377479:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204934.377480:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:2.0:1788204934.377485:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377495:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204934.377496:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204934.377498:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204934.377500:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205912320, offset 0 00000400:00000200:2.0:1788204934.377503:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204934.377505:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.377508:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204934.377510:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204934.377511:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204934.377513:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204934.377515:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204934.377517:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58983:58983:256:4294967295] 192.168.201.151@tcp LPNI seq info [58963:58963:8:4294967295] 00000400:00000200:2.0:1788204934.377519:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.377523:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204934.377525:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.377528:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204934.377530:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204934.377532:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204934.377535:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204934.377537:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1788204934.377541:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.377542:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204934.377545:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204934.377549:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204934.377552:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204934.377554:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204934.377556:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204934.377560:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377567:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.377570:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377576:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377578:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204934.377582:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204934.377586:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204934.377590:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204934.377592:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301061787 00000100:00000001:2.0:1788204934.377595:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.377597:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301061787 00000100:00000001:2.0:1788204934.377599:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.377601:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204934.377603:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377605:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204934.377609:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204934/real 1788204934] req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204934.377617:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204934.377618:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204934.377627:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204934.377631:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204934.377632:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204934.377634:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204934.377638:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204934.377646:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377652:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204934.377654:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377656:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377657:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204934.377677:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377691:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204934.377696:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377704:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204934.377706:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377707:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377709:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377711:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204934.377713:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204934.377716:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204934.377719:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204934.377721:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204934.377725:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204934.377733:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912320:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204934.377738:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204934.377741:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377748:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204934.377749:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.377752:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204934.377753:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204934.377755:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204934.377758:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912448:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204934.377762:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204934.377763:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204934.377765:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.377767:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204934.377771:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204934.377773:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.377777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204934.377780:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204934.377783:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204934.377786:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912448, portal 4 00000100:00000001:2.0:1788204934.377788:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204934.377789:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:2.0:1788204934.377795:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377802:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204934.377804:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204934.377807:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204934.377809:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205912448, offset 0 00000400:00000200:2.0:1788204934.377814:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204934.377817:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.377821:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204934.377824:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204934.377826:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204934.377828:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204934.377831:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204934.377834:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58984:58984:256:4294967295] 192.168.201.151@tcp LPNI seq info [58964:58964:8:4294967295] 00000400:00000200:2.0:1788204934.377838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.377844:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204934.377848:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.377851:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204934.377855:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204934.377858:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204934.377862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204934.377865:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1788204934.377869:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.377871:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204934.377874:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204934.377877:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204934.377880:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204934.377881:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204934.377885:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204934.377888:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377896:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.377899:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377904:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377906:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204934.377909:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204934.377912:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204934.377915:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204934.377917:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301061673 00000100:00000001:2.0:1788204934.377919:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.377922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301061673 00000100:00000001:2.0:1788204934.377924:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.377925:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204934.377927:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.377929:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204934.377932:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204934/real 1788204934] req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204934.377939:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204934.377940:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204934.377942:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204934.377944:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204934.377945:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204934.377947:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204934.377950:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204934.377957:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377968:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204934.377970:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377972:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377973:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204934.377977:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377983:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204934.377987:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.377993:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204934.377996:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.377997:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.377999:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378000:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204934.378003:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204934.378005:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204934.378008:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204934.378010:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204934.378014:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204934.378023:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912448:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204934.378027:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204934.378030:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378037:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204934.378039:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.378042:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204934.378044:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204934.378046:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204934.378049:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912576:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204934.378052:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204934.378054:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204934.378055:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204934.378057:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204934.378060:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204934.378062:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204934.378065:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204934.378067:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204934.378070:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204934.378073:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912576, portal 4 00000100:00000001:2.0:1788204934.378075:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204934.378076:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:2.0:1788204934.378081:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378087:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204934.378089:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204934.378091:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204934.378093:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205912576, offset 0 00000400:00000200:2.0:1788204934.378096:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204934.378099:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.378102:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204934.378104:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204934.378107:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204934.378109:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204934.378111:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204934.378114:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58985:58985:256:4294967295] 192.168.201.151@tcp LPNI seq info [58965:58965:8:4294967295] 00000400:00000200:2.0:1788204934.378117:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204934.378123:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204934.378128:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.378131:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204934.378135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204934.378137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1788204934.378141:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204934.378144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1788204934.378147:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204934.378149:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204934.378152:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204934.378155:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204934.378157:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204934.378159:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204934.378161:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204934.378165:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378172:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.378175:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204959 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378181:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.378183:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204934.378185:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204934.378188:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204934.378191:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204934.378193:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301061679 00000100:00000001:2.0:1788204934.378196:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.378198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301061679 00000100:00000001:2.0:1788204934.378200:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301065729 00000100:00000001:2.0:1788204934.378202:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204934.378204:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.378205:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204934.378209:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204934/real 1788204934] req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204934.378216:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204934.378217:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204934.378219:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204934.378221:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204934.378222:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204934.378224:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204934.378227:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204934.378235:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378241:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204934.378243:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378244:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.378246:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204934.378249:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378255:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204934.378259:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378266:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204934.378268:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378270:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204934.378271:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378273:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204934.378275:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204934.378277:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204934.378280:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204934.378283:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204934.378287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204934.378295:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912576:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204934.378299:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378300:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.378303:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205912320/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378310:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204934.378311:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204934.378313:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204934.378316:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204934.378317:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204934.378318:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204934.378320:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204934.378322:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204934.378324:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204934.378325:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204934.378326:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:2.0:1788204934.378328:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204934.378329:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378330:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.378333:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205912448/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378340:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204934.378341:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204934.378342:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204934.378345:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204934.378346:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204934.378347:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204934.378349:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204934.378350:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204934.378352:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204934.378354:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204934.378356:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:2.0:1788204934.378358:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204934.378359:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378360:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204934.378380:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205912576/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204959 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204934.378387:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204934.378389:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204934.378391:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d000. 02000000:00000001:2.0:1788204934.378393:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204934.378394:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204934.378396:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204934.378398:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204934.378399:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204934.378402:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204934.378403:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204934.378404:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:2.0:1788204934.378406:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204934.378408:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378409:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204934.378413:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204934.378414:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204934.378416:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204934.378418:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204934.568507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204934.568512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204934.568516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204934.568517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204934.568519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204934.568520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204934.568522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204934.568523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204934.568537:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204934.568539:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204934.568541:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204934.888483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204934.888489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204934.888492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204934.888498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204934.888501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204934.888505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.888507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204934.888510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.888513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204935.336488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204935.336495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204935.336497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204935.336502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204935.336505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204935.336509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204935.336511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204935.336513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204935.336517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204935.400423:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204935.400423:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204935.400431:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204935.400431:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204935.400435:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204935.400435:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204935.400442:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204935.400442:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204935.400447:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204935.400447:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204935.400452:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204935.400453:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204935.400457:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204935.400460:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204935.400462:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204935.400463:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204935.400466:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204935.400468:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204935.592480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204935.592487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204935.592493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204935.592495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204935.592498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204935.592499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204935.592501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204935.592503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204935.592563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204935.592567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204935.592571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204935.912502:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204935.912510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204935.912512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204935.912516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204935.912520:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204935.912524:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204935.912527:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204935.912530:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204935.912533:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.360422:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204936.360429:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204936.360431:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204936.360437:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204936.360440:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204936.360444:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204936.360446:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204936.360449:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204936.360451:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.424462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204936.424468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204936.424469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204936.424470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.424475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788204936.424477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204936.424478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.424483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204936.424484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204936.424488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204936.424489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204936.424493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204936.424494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.424498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.424503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.424505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204936.424509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204936.424512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204936.616512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204936.616518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204936.616523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204936.616524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204936.616527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204936.616528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204936.616531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204936.616532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204936.616554:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204936.616556:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204936.616558:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204936.936522:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204936.936529:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204936.936531:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204936.936537:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204936.936541:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204936.936545:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204936.936547:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204936.936550:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204936.936553:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.384477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204937.384484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204937.384486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204937.384491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204937.384495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204937.384498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.384501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204937.384503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.384506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204937.448400:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204937.448405:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204937.448407:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204937.448410:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204937.448413:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204937.448416:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204937.448419:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204937.448421:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204937.448423:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.448436:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204937.448440:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204937.448441:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204937.448443:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204937.448445:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204937.448447:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.448449:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204937.448450:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.448452:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204937.640467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204937.640474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204937.640479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204937.640480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204937.640482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204937.640483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204937.640485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204937.640486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204937.640551:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204937.640555:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204937.640559:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204937.960468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204937.960476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204937.960479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204937.960485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204937.960488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204937.960492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204937.960494:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204937.960497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204937.960501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.408460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204938.408466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204938.408467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204938.408471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204938.408473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204938.408475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204938.408477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204938.408479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204938.408480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.472496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204938.472496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204938.472509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204938.472510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204938.472520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204938.472520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204938.472531:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204938.472531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204938.472548:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204938.472548:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204938.472565:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204938.472565:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204938.472573:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204938.472576:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204938.472579:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.472581:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204938.472585:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204938.472602:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204938.664503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204938.664511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204938.664516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204938.664517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204938.664519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204938.664520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204938.664523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204938.664524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204938.664542:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204938.664545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204938.664547:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204938.984476:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301065729, at 4301066240 00000100:00000040:0.0:1788204938.984481:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301065729, at 4301066240 00000100:00000010:1.0:1788204938.984489:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000010:0.0:1788204938.984492:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00000001:3.0:1788204938.984493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204938.984497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204938.984499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204938.984502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204938.984505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204938.984508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204938.984510:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204938.984512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204938.984515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204939.432471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204939.432477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204939.432479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204939.432482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204939.432485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204939.432488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.432490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204939.432492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.432494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496514:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204939.496524:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1788204939.496525:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204939.496531:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204939.496531:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204939.496535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496537:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204939.496541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204939.496545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204939.496549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.496552:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204939.496552:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204939.496557:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.496560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496563:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6394 00000400:00000200:3.0:1788204939.496587:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204939.496593:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204939.496596:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204939.496599:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496602:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204939.496605:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496608:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204939.496612:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204939.496614:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788204939.496617:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204939.496620:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788204939.496623:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788204939.496626:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788204939.496629:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496632:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204939.496634:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496636:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204939.496639:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788204939.496643:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496645:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204939.496648:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204939.496651:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204939.496654:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496657:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204939.496659:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204939.496667:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2a00. 00000020:00000040:3.0:1788204939.496670:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788204939.496674:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204939.496676:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496679:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204939.496682:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204939.496686:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204939.496693:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23800. 00000100:00000001:3.0:1788204939.496699:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204939.496707:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204939.496718:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204939.496722:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204939.496727:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c42bbc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204939.496747:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496750:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204939.496753:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204939.496755:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204939.496757:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204939.496761:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204939.496764:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6394 00000400:00000200:3.0:1788204939.496768:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204939.496772:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1788204939.496772:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204939.496777:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788204939.496778:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204939.496781:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204939.496783:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496787:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204939.496792:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496794:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204939.496796:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204939.496797:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000040:2.0:1788204939.496798:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204939.496799:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204939.496803:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788204939.496807:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788204939.496812:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788204939.496816:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788204939.496818:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204939.496819:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496821:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788204939.496826:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1788204939.496826:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496829:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788204939.496831:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1788204939.496833:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788204939.496836:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496838:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496843:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.496844:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205912704:192.168.201.151@tcp:250:kworker.0 02000000:00000001:3.0:1788204939.496847:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788204939.496851:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204939.496855:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204939.496857:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204939.496859:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.496861:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496863:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1788204939.496865:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1788204939.496869:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000010:2.0:1788204939.496872:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 00000008:00000001:3.0:1788204939.496873:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.496877:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496878:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204939.496884:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0a80. 00000400:00000200:2.0:1788204939.496885:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:3.0:1788204939.496889:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:2.0:1788204939.496892:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 02000000:00000001:3.0:1788204939.496893:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204939.496897:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.496900:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204939.496901:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1788204939.496902:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 02000000:00000001:3.0:1788204939.496904:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204939.496910:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912704, portal 25 02000000:00000010:3.0:1788204939.496911:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21c00. 00000100:00000001:2.0:1788204939.496915:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204939.496918:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000001:3.0:1788204939.496918:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204939.496931:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1788204939.496938:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.496950:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204939.496955:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496956:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1788204939.496960:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000001:3.0:1788204939.496960:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788204939.496967:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205912704, offset 0 00000100:00000040:3.0:1788204939.496969:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed2b742] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204939.496979:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204939.496984:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.496987:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204939.496990:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1788204939.496991:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.496997:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:3.0:1788204939.496997:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204939.497002:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204939.497004:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204939.497008:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788204939.497011:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:3.0:1788204939.497015:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6394 00000400:00000200:2.0:1788204939.497019:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204939.497024:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204939.497028:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58986:58986:256:4294967295] 192.168.201.151@tcp LPNI seq info [58966:58966:8:4294967295] 00000400:00000200:2.0:1788204939.497033:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204939.497038:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204939.497047:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:2.0:1788204939.497050:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:3.0:1788204939.497050:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204939.497057:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497059:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204939.497062:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.0:1788204939.497064:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497065:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204939.497069:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204939.497071:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788204939.497072:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204939.497073:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000010:2.0:1788204939.497074:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000020:00000001:3.0:1788204939.497075:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788204939.497079:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788204939.497081:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497084:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1788204939.497086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:3.0:1788204939.497088:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497091:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:2.0:1788204939.497093:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000020:00000040:3.0:1788204939.497093:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788204939.497097:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497100:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204939.497101:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204939.497104:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1788204939.497105:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:3.0:1788204939.497106:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497109:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204939.497113:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.1:1788204939.497113:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000008:00000001:3.0:1788204939.497115:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497116:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204939.497119:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0700. 00000020:00000040:3.0:1788204939.497121:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788204939.497122:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204939.497123:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497124:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204939.497125:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204939.497126:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204939.497130:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22c00. 00000100:00000001:3.0:1788204939.497132:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204939.497136:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204939.497144:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:2.0:1788204939.497145:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497146:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788204939.497151:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000100:00000040:3.0:1788204939.497152:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b25c632d] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204939.497159:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204939.497160:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1788204939.497162:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204939.497166:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204939.497167:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204939.497169:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788204939.497171:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788204939.497172:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204939.497177:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6394 00000400:00000200:3.0:1788204939.497179:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.497181:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000200:3.0:1788204939.497182:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:2.0:1788204939.497186:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00080000:3.0:1788204939.497186:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204939.497192:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497193:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204939.497194:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788204939.497195:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:2.0:1788204939.497202:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204939.497205:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788204939.497210:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788204939.497213:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788204939.497216:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788204939.497220:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:2.0:1788204939.497221:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788204939.497224:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204939.497228:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204939.497229:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788204939.497237:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204939.497244:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497244:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204939.497248:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1788204939.497249:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497253:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788204939.497255:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:2.0:1788204939.497257:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204939.497258:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204939.497262:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204939.497264:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788204939.497265:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788204939.497267:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204939.497269:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301070849 02000000:00000001:3.0:1788204939.497270:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497275:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204939.497276:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000010:2.0:1788204939.497279:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1ba0. 00000008:00000001:3.0:1788204939.497279:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497283:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204939.497286:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204939.497288:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000040:3.0:1788204939.497288:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788204939.497293:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204939.497294:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.497295:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204939.497297:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788204939.497298:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301070849 02000000:00000001:3.0:1788204939.497299:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204939.497303:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21000. 00000100:00000010:2.0:1788204939.497306:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1300. 00000100:00000001:3.0:1788204939.497307:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497312:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788204939.497313:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204939.497319:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204939.497320:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204939.497324:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497326:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204939.497328:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bfc2cda] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497330:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788204939.497337:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.497339:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204939/real 1788204939] req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204939.497340:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6404) 00000400:00000001:2.0:1788204939.497352:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204939.497354:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204939.497357:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204939.497360:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204939.497394:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204939.497397:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204939.497408:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204939.497418:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497425:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204939.497428:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497430:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204939.497431:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204939.497433:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.497434:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204939.497437:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.497441:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204939.497442:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204939.497446:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204939.497450:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.497456:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204939.497456:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.497458:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204939.497462:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204939.497464:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497472:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204939.497475:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497477:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.497479:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497482:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204939.497484:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204939.497487:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204939.497491:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204939.497494:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204939.497498:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204939.497506:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205912704:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204939.497511:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497513:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.497515:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205912704/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497521:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204939.497523:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204939.497525:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204939.497528:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204939.497529:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204939.497531:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788204939.497533:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204939.497534:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:2.0:1788204939.497538:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204939.497539:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204939.497541:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:2.0:1788204939.497544:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204939.497545:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497547:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497553:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204939.497554:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497556:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204939.497558:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204939.497559:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204939.497563:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497569:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204939.497570:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.497588:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204939.497590:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204939.497592:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.497594:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912832:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204939.497596:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204939.497598:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204939.497599:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.497600:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204939.497603:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204939.497604:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204939.497606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204939.497608:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204939.497610:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204939.497612:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912832, portal 10 00000100:00000001:2.0:1788204939.497614:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204939.497615:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:2.0:1788204939.497619:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497623:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204939.497624:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204939.497625:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204939.497627:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205912832, offset 0 00000400:00000200:2.0:1788204939.497629:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204939.497632:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.497635:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204939.497637:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204939.497638:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204939.497640:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204939.497641:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204939.497643:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58987:58987:256:4294967295] 192.168.201.151@tcp LPNI seq info [58967:58967:8:4294967295] 00000400:00000200:2.0:1788204939.497646:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.497650:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204939.497652:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.497655:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204939.497657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204939.497659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204939.497662:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204939.497663:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204939.497665:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.497667:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204939.497669:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204939.497671:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204939.497673:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204939.497674:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204939.497675:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204939.497677:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497682:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.497684:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497687:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497688:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204939.497690:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204939.497692:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788204939.497694:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204939.497695:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301066907 00000100:00000001:2.0:1788204939.497696:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.497698:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301066907 00000100:00000001:2.0:1788204939.497699:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.497700:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204939.497702:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497703:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204939.497705:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204939/real 1788204939] req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204939.497713:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204939.497714:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204939.497715:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204939.497716:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204939.497717:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204939.497718:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204939.497720:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204939.497724:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497727:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204939.497728:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497729:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.497730:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204939.497732:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497736:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204939.497738:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497741:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204939.497743:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497743:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.497744:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497746:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204939.497748:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204939.497750:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204939.497753:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204939.497755:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204939.497759:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204939.497766:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912832:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788204939.497769:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204939.497772:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497778:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204939.497780:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.497782:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204939.497783:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204939.497785:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.497788:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204939.497791:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204939.497793:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204939.497794:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.497796:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204939.497799:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204939.497800:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204939.497804:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204939.497806:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204939.497809:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204939.497812:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205912960, portal 4 00000100:00000001:2.0:1788204939.497814:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204939.497815:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00000040:2.0:1788204939.497820:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497826:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204939.497828:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204939.497830:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204939.497832:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205912960, offset 0 00000400:00000200:2.0:1788204939.497835:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204939.497838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.497840:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204939.497842:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204939.497843:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204939.497845:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204939.497846:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204939.497848:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58988:58988:256:4294967295] 192.168.201.151@tcp LPNI seq info [58968:58968:8:4294967295] 00000400:00000200:2.0:1788204939.497851:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.497854:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204939.497857:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.497859:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204939.497861:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204939.497863:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204939.497865:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204939.497867:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204939.497869:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.497870:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204939.497872:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204939.497874:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204939.497876:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204939.497877:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204939.497879:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204939.497881:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497888:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.497891:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497896:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497898:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204939.497901:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204939.497904:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788204939.497907:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204939.497910:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301066793 00000100:00000001:2.0:1788204939.497912:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.497914:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301066793 00000100:00000001:2.0:1788204939.497915:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.497917:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204939.497919:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.497921:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204939.497924:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204939/real 1788204939] req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204939.497931:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204939.497932:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204939.497934:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204939.497936:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204939.497938:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204939.497940:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204939.497943:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204939.497951:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497955:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204939.497956:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497958:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.497958:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204939.497961:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497964:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204939.497967:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497970:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204939.497972:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497973:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.497973:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.497974:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204939.497976:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204939.497977:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204939.497979:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204939.497980:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204939.497982:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204939.497986:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205912960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204939.497988:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204939.497990:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.497994:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204939.497995:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.497996:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204939.497997:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204939.497998:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204939.498001:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913088:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204939.498004:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204939.498005:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204939.498006:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204939.498008:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788204939.498011:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204939.498013:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204939.498016:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788204939.498018:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788204939.498021:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788204939.498023:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913088, portal 4 00000100:00000001:2.0:1788204939.498025:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204939.498026:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:2.0:1788204939.498031:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498038:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204939.498040:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788204939.498042:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788204939.498044:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205913088, offset 0 00000400:00000200:2.0:1788204939.498047:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204939.498051:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.498054:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788204939.498057:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788204939.498059:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788204939.498061:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204939.498063:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788204939.498066:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58989:58989:256:4294967295] 192.168.201.151@tcp LPNI seq info [58969:58969:8:4294967295] 00000400:00000200:2.0:1788204939.498069:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204939.498075:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788204939.498079:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.498082:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788204939.498086:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788204939.498088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1788204939.498092:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788204939.498095:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1788204939.498099:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788204939.498100:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788204939.498103:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788204939.498106:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788204939.498109:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788204939.498111:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788204939.498113:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204939.498116:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498126:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.498128:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498132:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.498133:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204939.498135:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204939.498137:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788204939.498140:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204939.498142:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301066799 00000100:00000001:2.0:1788204939.498143:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.498145:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301066799 00000100:00000001:2.0:1788204939.498147:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301070849 00000100:00000001:2.0:1788204939.498148:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204939.498149:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.498151:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204939.498154:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204939/real 1788204939] req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788204939.498159:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788204939.498160:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788204939.498161:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788204939.498162:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788204939.498163:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788204939.498165:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788204939.498167:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788204939.498171:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498175:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204939.498177:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498178:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.498178:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204939.498180:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498184:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204939.498186:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498189:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204939.498191:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498191:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204939.498192:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498193:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204939.498194:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204939.498196:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788204939.498197:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788204939.498198:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788204939.498200:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204939.498204:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913088:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788204939.498206:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498207:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.498209:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205912832/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498213:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204939.498214:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204939.498215:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788204939.498216:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204939.498217:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204939.498218:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788204939.498220:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204939.498221:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:2.0:1788204939.498223:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204939.498224:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204939.498225:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:2.0:1788204939.498227:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204939.498228:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498229:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.498232:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205912960/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498236:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204939.498237:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204939.498238:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0c400. 02000000:00000001:2.0:1788204939.498240:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204939.498241:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204939.498242:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788204939.498243:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204939.498243:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:2.0:1788204939.498245:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204939.498245:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204939.498246:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:2.0:1788204939.498247:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204939.498248:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498249:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204939.498250:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205913088/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204939.498253:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788204939.498254:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788204939.498254:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788204939.498256:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788204939.498256:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788204939.498257:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788204939.498258:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788204939.498258:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788204939.498260:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788204939.498261:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788204939.498262:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:2.0:1788204939.498264:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788204939.498265:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498267:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204939.498270:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204939.498272:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204939.498274:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204939.498275:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204939.688456:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204939.688462:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204939.688467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204939.688468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204939.688471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204939.688473:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204939.688475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204939.688477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204939.688533:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204939.688537:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204939.688540:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204940.008482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204940.008489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204940.008491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204940.008497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204940.008501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204940.008504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204940.008507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204940.008511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204940.008514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.456469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204940.456477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204940.456479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204940.456484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204940.456487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204940.456491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204940.456493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204940.456496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204940.456499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.520452:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204940.520452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204940.520458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204940.520459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204940.520461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.520462:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204940.520465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204940.520467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204940.520469:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204940.520473:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204940.520474:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204940.520477:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204940.520479:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204940.520481:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.520484:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204940.520489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204940.520490:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204940.520493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204940.712507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204940.712513:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204940.712515:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204940.712517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712519:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204940.712521:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204940.712526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204940.712528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204940.712529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204940.712539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204940.712541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204940.712543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204941.032470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204941.032478:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204941.032480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.032485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204941.032489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204941.032493:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.032495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204941.032497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.032500:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.480487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204941.480494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204941.480497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204941.480502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204941.480506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204941.480510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204941.480512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204941.480514:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204941.480517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.544479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204941.544479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204941.544487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204941.544488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204941.544492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.544492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204941.544499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204941.544500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204941.544505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204941.544505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204941.544510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204941.544511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204941.544515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204941.544519:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204941.544524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.544524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204941.544528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204941.544532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204941.736479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204941.736486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204941.736492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204941.736494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204941.736504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204941.736506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204941.736509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204941.736512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204941.736585:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204941.736591:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204941.736596:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204942.056466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204942.056476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204942.056478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204942.056485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204942.056490:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204942.056495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204942.056498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204942.056501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204942.056505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204942.504457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204942.504467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204942.504469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204942.504474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204942.504477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204942.504481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204942.504483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204942.504485:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204942.504488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204942.568492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204942.568500:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204942.568501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204942.568504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204942.568509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788204942.568510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204942.568515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204942.568516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204942.568536:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204942.568538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204942.568542:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204942.568547:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204942.568548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204942.568557:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204942.568561:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204942.568564:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204942.568567:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204942.568570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204942.760561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204942.760568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204942.760573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204942.760574:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204942.760578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204942.760579:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204942.760582:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204942.760583:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204942.760603:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204942.760605:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204942.760608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204943.080488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204943.080497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204943.080499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204943.080504:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204943.080507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204943.080533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204943.080536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204943.080539:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204943.080542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204943.528460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204943.528466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204943.528467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204943.528471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204943.528477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204943.528480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.528482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204943.528484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.528486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204943.592459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204943.592459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204943.592466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204943.592466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204943.592469:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204943.592469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204943.592475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:2.0:1788204943.592475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204943.592479:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204943.592479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204943.592483:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204943.592483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.592486:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204943.592488:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.592490:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204943.592492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204943.592494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204943.592496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204943.784515:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204943.784517:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204943.784520:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784523:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204943.784525:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784526:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784529:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204943.784535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204943.784539:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204943.784541:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204943.784543:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784544:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204943.784545:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784547:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784548:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204943.784553:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204943.784554:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204943.784554:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784555:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204943.784556:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784557:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204943.784558:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204943.784612:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204943.784616:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204943.784618:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204944.104466:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301070849, at 4301071360 00000100:00000001:3.0:1788204944.104467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204944.104468:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301070849, at 4301071360 00000100:00000001:3.0:1788204944.104474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204944.104477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204944.104480:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1300. 00000100:00000010:0.0:1788204944.104482:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1ba0. 00000100:00100000:3.0:1788204944.104482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204944.104488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204944.104508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.104511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.104515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.104519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204944.552435:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204944.552442:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204944.552444:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204944.552450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204944.552453:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204944.552457:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.552459:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204944.552461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.552464:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204944.616475:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204944.616475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204944.616482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204944.616497:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204944.616497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616502:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788204944.616503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204944.616507:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204944.616508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204944.616512:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204944.616513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.616517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.616520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204944.616522:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6399 00000100:00000001:3.0:1788204944.616523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204944.616533:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204944.616540:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204944.616543:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204944.616547:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616550:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204944.616552:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616555:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204944.616558:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204944.616559:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204944.616562:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204944.616565:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204944.616567:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204944.616570:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204944.616574:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616576:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204944.616578:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616581:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204944.616584:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204944.616587:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616591:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.616594:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204944.616595:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204944.616598:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788204944.616598:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204944.616600:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204944.616604:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 02000000:00000001:2.0:1788204944.616605:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204944.616606:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204944.616609:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204944.616609:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204944.616612:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204944.616613:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204944.616614:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.616617:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204944.616621:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204944.616625:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204944.616628:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204944.616629:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616631:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204944.616632:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204944.616636:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204944.616641:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204944.616646:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204944.616653:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204944.616667:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204944.616670:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204944.616674:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204944.616688:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204944.616690:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204944.616692:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204944.616693:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204944.616694:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204944.616696:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204944.616697:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6399 00000400:00000200:2.0:1788204944.616700:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204944.616702:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204944.616703:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204944.616705:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616708:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204944.616709:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616711:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204944.616712:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788204944.616712:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1788204944.616716:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204944.616717:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204944.616717:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1788204944.616720:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204944.616721:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204944.616722:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204944.616725:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204944.616730:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616731:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204944.616731:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204944.616735:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616738:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204944.616742:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204944.616746:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616750:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.616753:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204944.616753:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.616756:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.616759:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204944.616760:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204944.616763:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204944.616764:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616768:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204944.616768:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204944.616773:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204944.616773:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205913216:192.168.201.151@tcp:250:kworker.0 00000008:00000001:2.0:1788204944.616779:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.616780:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204944.616781:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.616783:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:2.0:1788204944.616785:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 02000000:00000001:3.0:1788204944.616786:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204944.616790:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204944.616790:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204944.616794:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204944.616795:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616796:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:3.0:1788204944.616796:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204944.616799:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.616800:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204944.616803:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.616806:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788204944.616807:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204944.616811:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204944.616812:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00100000:2.0:1788204944.616816:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204944.616819:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:2.0:1788204944.616823:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204944.616825:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204944.616828:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204944.616828:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913216, portal 25 00000100:00000001:3.0:1788204944.616834:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204944.616837:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000001:2.0:1788204944.616838:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204944.616843:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204944.616845:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204944.616847:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:3.0:1788204944.616849:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204944.616854:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204944.616860:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204944.616860:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204944.616863:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6399 00000100:00000040:3.0:1788204944.616865:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204944.616868:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204944.616868:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000400:00000200:2.0:1788204944.616873:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000200:3.0:1788204944.616873:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205913216, offset 0 00000100:00080000:2.0:1788204944.616877:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204944.616880:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616883:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204944.616884:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204944.616887:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616889:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204944.616891:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204944.616892:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204944.616894:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204944.616895:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788204944.616896:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204944.616899:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204944.616900:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204944.616901:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:2.0:1788204944.616905:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616907:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204944.616909:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788204944.616910:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616913:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204944.616914:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:2.0:1788204944.616916:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204944.616920:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788204944.616921:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204944.616926:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.616932:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.616933:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58990:58990:256:4294967295] 192.168.201.151@tcp LPNI seq info [58970:58970:8:4294967295] 02000000:00000001:2.0:1788204944.616934:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204944.616937:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204944.616940:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616943:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204944.616944:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788204944.616945:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204944.616949:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.616950:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204944.616952:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204944.616956:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204944.616960:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204944.616961:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.616962:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788204944.616963:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204944.616965:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204944.616968:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204944.616972:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000010:2.0:1788204944.616975:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000800:00000010:3.0:1788204944.616977:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000001:2.0:1788204944.616981:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204944.616985:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204944.616989:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00100000:2.0:1788204944.616990:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204944.616996:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204944.617000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204944.617006:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204944.617008:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1788204944.617009:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000100:00000001:2.0:1788204944.617013:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.617014:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788204944.617020:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204944.617020:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204944.617026:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204944.617030:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:2.0:1788204944.617031:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204944.617034:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204944.617034:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788204944.617038:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:3.0:1788204944.617040:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204944.617041:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204944.617043:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204944.617048:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204944.617052:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6399 00000100:00000001:3.0:1788204944.617054:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1788204944.617057:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:3.0:1788204944.617059:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204944.617062:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204944.617066:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204944.617071:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204944.617072:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.617075:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204944.617076:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1788204944.617078:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.617081:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204944.617082:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204944.617083:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204944.617084:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204944.617085:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204944.617086:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204944.617090:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204944.617092:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:3.0:1788204944.617094:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788204944.617096:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204944.617099:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204944.617100:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204944.617100:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301075969 00000100:00000001:2.0:1788204944.617104:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204944.617106:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000010:3.0:1788204944.617108:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f43c0. 00000100:00080000:2.0:1788204944.617109:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204944.617113:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.617114:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204944.617116:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204944.617118:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617118:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204944.617119:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204944.617121:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204944.617124:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.617125:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617127:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301075969 00000100:00000010:2.0:1788204944.617128:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000010:3.0:1788204944.617131:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4cc0. 00000020:00000040:2.0:1788204944.617132:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204944.617135:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788204944.617136:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204944.617138:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204944.617140:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204944.617140:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204944.617143:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204944.617144:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204944.617146:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617147:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000010:2.0:1788204944.617151:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00100000:3.0:1788204944.617153:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204944/real 1788204944] req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204944.617155:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204944.617162:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788204944.617169:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788204944.617170:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204944.617174:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204944.617174:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1788204944.617179:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204944.617179:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204944.617185:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000001:2.0:1788204944.617186:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204944.617189:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000040:2.0:1788204944.617190:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6409) 00000100:00000001:3.0:1788204944.617194:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204944.617200:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204944.617208:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617213:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204944.617214:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617216:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617217:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204944.617219:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617223:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204944.617226:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617230:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204944.617231:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617232:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617234:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617235:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204944.617236:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204944.617238:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204944.617241:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204944.617242:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204944.617244:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204944.617249:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205913216:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204944.617251:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617252:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617254:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205913216/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617258:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204944.617260:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617261:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204944.617264:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204944.617265:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204944.617266:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204944.617267:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204944.617269:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788204944.617271:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204944.617272:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204944.617273:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788204944.617275:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204944.617276:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617278:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617283:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204944.617284:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617285:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204944.617286:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204944.617287:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204944.617289:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617293:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.617294:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617295:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204944.617297:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204944.617298:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204944.617299:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913344:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204944.617301:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204944.617302:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204944.617303:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617304:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617306:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204944.617308:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.617310:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204944.617312:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204944.617314:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204944.617316:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913344, portal 10 00000100:00000001:3.0:1788204944.617317:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204944.617318:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788204944.617322:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617326:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204944.617327:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204944.617329:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204944.617330:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205913344, offset 0 00000400:00000200:3.0:1788204944.617332:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617334:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617337:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204944.617338:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204944.617340:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204944.617342:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204944.617343:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617345:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58991:58991:256:4294967295] 192.168.201.151@tcp LPNI seq info [58971:58971:8:4294967295] 00000400:00000200:3.0:1788204944.617347:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617351:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204944.617354:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617356:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204944.617358:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204944.617360:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204944.617379:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204944.617381:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204944.617383:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617385:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204944.617386:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204944.617389:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204944.617390:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204944.617392:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204944.617393:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204944.617395:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617400:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617402:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617406:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617406:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204944.617408:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204944.617411:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204944.617413:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204944.617414:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301072027 00000100:00000001:3.0:1788204944.617416:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617417:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301072027 00000100:00000001:3.0:1788204944.617419:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617420:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204944.617421:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617422:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204944.617425:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204944/real 1788204944] req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204944.617429:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204944.617430:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204944.617431:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204944.617432:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204944.617433:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204944.617434:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204944.617435:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204944.617440:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617444:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204944.617444:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617445:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617446:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204944.617448:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617451:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204944.617454:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617460:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204944.617462:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617464:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617465:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617466:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204944.617468:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204944.617470:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204944.617473:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204944.617475:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204944.617479:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204944.617501:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913344:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204944.617505:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204944.617509:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617514:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.617515:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617517:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204944.617518:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204944.617520:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204944.617521:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913472:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204944.617523:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204944.617524:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204944.617525:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617526:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617529:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204944.617530:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.617532:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204944.617534:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204944.617536:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204944.617538:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913472, portal 4 00000100:00000001:3.0:1788204944.617539:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204944.617540:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788204944.617544:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617548:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204944.617549:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204944.617550:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204944.617552:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205913472, offset 0 00000400:00000200:3.0:1788204944.617554:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617556:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617559:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204944.617560:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204944.617562:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204944.617563:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204944.617565:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617567:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58992:58992:256:4294967295] 192.168.201.151@tcp LPNI seq info [58972:58972:8:4294967295] 00000400:00000200:3.0:1788204944.617569:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617575:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204944.617579:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617582:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204944.617585:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204944.617587:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204944.617592:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204944.617594:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204944.617598:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617599:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204944.617602:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204944.617605:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204944.617608:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204944.617610:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204944.617612:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204944.617615:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617621:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617623:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617627:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617628:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204944.617630:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204944.617633:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204944.617635:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204944.617636:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301071913 00000100:00000001:3.0:1788204944.617637:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617639:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301071913 00000100:00000001:3.0:1788204944.617640:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617641:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204944.617642:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617643:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204944.617645:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204944/real 1788204944] req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204944.617650:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204944.617651:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204944.617652:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204944.617653:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204944.617654:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204944.617655:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204944.617658:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204944.617665:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617671:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204944.617672:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617674:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617675:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204944.617678:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617684:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204944.617688:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617695:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204944.617697:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617698:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617700:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617702:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204944.617704:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204944.617706:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204944.617709:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204944.617711:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204944.617715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204944.617722:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913472:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204944.617726:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204944.617729:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617735:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.617737:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617739:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204944.617741:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204944.617743:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204944.617745:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913600:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204944.617748:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204944.617750:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204944.617751:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204944.617752:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617754:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204944.617755:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204944.617758:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204944.617759:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204944.617761:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204944.617763:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913600, portal 4 00000100:00000001:3.0:1788204944.617764:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204944.617765:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204944.617769:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617773:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204944.617774:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204944.617775:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204944.617776:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205913600, offset 0 00000400:00000200:3.0:1788204944.617778:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617783:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204944.617784:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204944.617786:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204944.617787:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204944.617789:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204944.617791:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58993:58993:256:4294967295] 192.168.201.151@tcp LPNI seq info [58973:58973:8:4294967295] 00000400:00000200:3.0:1788204944.617793:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204944.617796:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204944.617799:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617801:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204944.617803:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204944.617805:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204944.617808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204944.617810:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204944.617829:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204944.617831:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204944.617833:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204944.617836:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204944.617839:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204944.617841:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204944.617844:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204944.617847:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617855:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617857:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617861:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617862:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204944.617864:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204944.617866:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204944.617868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204944.617869:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301071919 00000100:00000001:3.0:1788204944.617871:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617872:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301071919 00000100:00000001:3.0:1788204944.617873:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301075969 00000100:00000001:3.0:1788204944.617874:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204944.617875:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.617877:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204944.617879:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204944/real 1788204944] req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204944.617883:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204944.617884:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204944.617885:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204944.617886:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204944.617887:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204944.617888:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204944.617890:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204944.617895:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617902:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204944.617903:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617904:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617905:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204944.617907:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617910:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204944.617912:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617915:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204944.617917:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617918:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204944.617918:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617920:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204944.617921:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204944.617924:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204944.617927:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204944.617929:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204944.617932:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204944.617939:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913600:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204944.617943:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617944:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617948:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205913344/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617954:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204944.617956:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617958:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204944.617960:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204944.617962:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204944.617963:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204944.617964:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204944.617966:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788204944.617968:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204944.617969:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204944.617970:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788204944.617971:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204944.617972:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617973:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617976:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205913472/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617980:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204944.617981:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617981:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204944.617983:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204944.617983:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204944.617984:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204944.617985:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204944.617986:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:3.0:1788204944.617987:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204944.617987:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204944.617988:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788204944.617989:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204944.617990:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.617991:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204944.617992:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205913600/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204944.617996:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204944.617996:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204944.617997:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204944.617998:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204944.617999:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204944.617999:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204944.618001:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204944.618001:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:3.0:1788204944.618002:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204944.618003:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204944.618004:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204944.618005:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204944.618006:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.618007:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.618009:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204944.618010:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.618011:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204944.618012:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204944.808531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204944.808536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204944.808539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204944.808540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204944.808542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204944.808543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204944.808545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204944.808546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204944.808558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204944.808560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204944.808562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204945.128466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204945.128472:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204945.128485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204945.128490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204945.128493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204945.128495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.128497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204945.128499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.128501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204945.576492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204945.576498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204945.576499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204945.576503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204945.576506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204945.576508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204945.576510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204945.576512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204945.576513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204945.640460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204945.640460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204945.640475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204945.640475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204945.640479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204945.640479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204945.640484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204945.640485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204945.640487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204945.640490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204945.640491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.640494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204945.640495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.640496:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.640501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204945.640502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204945.640505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204945.640508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204945.832490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204945.832497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204945.832503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204945.832504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204945.832506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204945.832507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204945.832510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204945.832510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204945.832570:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204945.832573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204945.832577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204946.152509:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204946.152516:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.152518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204946.152523:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204946.152527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204946.152531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.152534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.152536:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.152540:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204946.600511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204946.600519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204946.600521:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204946.600527:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204946.600530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204946.600534:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204946.600537:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204946.600539:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204946.600541:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204946.664421:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204946.664430:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.664430:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204946.664432:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.664433:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.664434:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204946.664437:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204946.664437:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204946.664441:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204946.664441:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204946.664445:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.664445:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204946.664459:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204946.664462:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204946.664465:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.664465:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.664466:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.664468:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204946.856506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204946.856512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204946.856516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204946.856517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204946.856519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204946.856520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204946.856522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204946.856523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204946.856535:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204946.856537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204946.856538:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204947.176498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204947.176505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204947.176507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204947.176512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204947.176515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204947.176519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.176522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204947.176525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.176528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.624486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204947.624492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204947.624493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204947.624497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204947.624499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204947.624502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204947.624503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204947.624505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204947.624507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204947.688439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204947.688455:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204947.688461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204947.688469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204947.688474:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204947.688479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.688482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204947.688486:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.688490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.688493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204947.688502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204947.688504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204947.688509:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204947.688513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204947.688517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204947.688520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204947.688523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204947.688525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204947.880497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204947.880503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204947.880507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204947.880508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204947.880510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204947.880512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204947.880515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204947.880516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204947.880568:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204947.880573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204947.880576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204948.200513:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204948.200522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204948.200524:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204948.200528:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204948.200532:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204948.200535:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.200538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204948.200541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.200544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204948.648481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204948.648486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204948.648487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204948.648491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204948.648493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204948.648496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204948.648498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204948.648500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204948.648502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204948.712511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204948.712514:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204948.712517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204948.712521:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204948.712523:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204948.712527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204948.712528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204948.712532:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204948.712538:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204948.712539:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204948.712541:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204948.712547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.712549:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204948.712550:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204948.712557:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.712562:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204948.712564:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.712567:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204948.904516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204948.904520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204948.904523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204948.904524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204948.904526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204948.904527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204948.904529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204948.904530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204948.904545:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204948.904547:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204948.904548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204949.224481:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301075969, at 4301076480 00000100:00000040:1.0:1788204949.224482:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301075969, at 4301076480 00000100:00000001:3.0:1788204949.224482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204949.224498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788204949.224499:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000010:0.0:1788204949.224501:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000001:3.0:1788204949.224502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204949.224509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204949.224513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204949.224518:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.224522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204949.224525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.224529:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204949.672543:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204949.672553:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204949.672556:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204949.672562:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204949.672566:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204949.672572:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204949.672576:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204949.672578:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204949.672582:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204949.736490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204949.736490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204949.736499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204949.736499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204949.736500:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204949.736502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204949.736505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204949.736509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204949.736510:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1788204949.736511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204949.736514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204949.736515:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204949.736516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204949.736520:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204949.736521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.736521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204949.736526:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204949.736528:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204949.736532:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204949.736534:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6404 00000100:00000001:1.0:1788204949.736538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.736538:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204949.736542:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204949.736544:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204949.736547:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204949.736552:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204949.736555:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204949.736558:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736561:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204949.736565:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736568:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204949.736572:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204949.736580:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204949.736584:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204949.736587:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204949.736590:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204949.736592:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204949.736596:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736599:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204949.736600:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736603:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204949.736606:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204949.736609:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736611:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204949.736614:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204949.736617:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204949.736619:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.736621:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204949.736623:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204949.736632:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204949.736635:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204949.736639:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204949.736641:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.736644:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204949.736646:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204949.736649:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204949.736657:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204949.736663:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204949.736671:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204949.736679:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204949.736682:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204949.736686:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204949.736703:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204949.736706:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204949.736708:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204949.736710:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204949.736711:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204949.736713:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204949.736715:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6404 00000400:00000200:2.0:1788204949.736718:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204949.736720:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204949.736722:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204949.736724:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736726:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204949.736727:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204949.736728:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736731:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204949.736732:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788204949.736733:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204949.736736:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204949.736737:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204949.736738:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204949.736740:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204949.736742:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204949.736744:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204949.736748:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204949.736748:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204949.736751:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204949.736753:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736757:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204949.736760:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204949.736764:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204949.736765:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204949.736766:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.736768:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204949.736769:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204949.736772:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204949.736775:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204949.736775:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204949.736780:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.736781:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.736783:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1788204949.736786:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205913728:192.168.201.151@tcp:250:kworker.0 00000008:00100000:2.0:1788204949.736789:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204949.736792:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788204949.736796:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.736796:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788204949.736800:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.736800:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.736805:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1788204949.736806:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788204949.736808:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204949.736810:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204949.736811:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204949.736811:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000001:2.0:1788204949.736816:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204949.736816:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204949.736819:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204949.736821:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.736824:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788204949.736826:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000400:00000010:3.0:1788204949.736828:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:2.0:1788204949.736831:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204949.736836:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00100000:2.0:1788204949.736841:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204949.736844:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913728, portal 25 00000100:00000001:3.0:1788204949.736849:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204949.736850:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788204949.736856:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788204949.736860:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204949.736861:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.736870:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204949.736871:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204949.736874:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204949.736878:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204949.736882:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205913728, offset 0 00000100:00000001:2.0:1788204949.736890:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.736891:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204949.736893:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204949.736897:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204949.736898:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204949.736899:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204949.736902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204949.736903:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204949.736907:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6404 00000400:00000200:3.0:1788204949.736910:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204949.736915:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204949.736919:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204949.736923:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204949.736925:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204949.736928:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58994:58994:256:4294967295] 192.168.201.151@tcp LPNI seq info [58974:58974:8:4294967295] 00000400:00000200:3.0:1788204949.736931:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204949.736935:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204949.736941:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204949.736943:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788204949.736944:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204949.736951:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.736955:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204949.736955:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204949.736960:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000010:3.0:1788204949.736962:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000001:2.0:1788204949.736963:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204949.736966:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204949.736966:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204949.736968:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204949.736971:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000200:3.0:1788204949.736971:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788204949.736976:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.0:1788204949.736977:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1788204949.736978:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.1:1788204949.736983:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:2.0:1788204949.736988:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204949.736988:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788204949.736992:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788204949.736996:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204949.736998:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000010:3.0:1788204949.736999:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000100:00000001:2.0:1788204949.737002:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204949.737005:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204949.737006:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1788204949.737008:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204949.737010:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204949.737013:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204949.737013:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1788204949.737017:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204949.737019:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 02000000:00000001:2.0:1788204949.737020:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204949.737022:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:2.0:1788204949.737023:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204949.737027:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:3.0:1788204949.737028:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1788204949.737030:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.737033:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204949.737033:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204949.737037:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204949.737041:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:3.0:1788204949.737044:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788204949.737045:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204949.737047:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.737049:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1788204949.737049:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204949.737052:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204949.737054:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204949.737059:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:3.0:1788204949.737060:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.737064:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737064:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1788204949.737070:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204949.737073:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.737082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204949.737085:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204949.737087:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301081089 00000100:00000001:2.0:1788204949.737088:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204949.737093:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204949.737097:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f47e0. 00000100:00000001:2.0:1788204949.737098:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204949.737102:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204949.737104:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204949.737106:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204949.737107:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204949.737109:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204949.737111:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6404 00000400:00000200:2.0:1788204949.737113:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204949.737113:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000400:00000200:2.0:1788204949.737115:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204949.737118:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204949.737120:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.737121:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204949.737123:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204949.737124:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204949.737126:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204949.737127:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788204949.737127:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301081089 00000020:00000001:2.0:1788204949.737130:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204949.737133:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204949.737134:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000010:3.0:1788204949.737134:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4360. 00000020:00000001:2.0:1788204949.737140:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204949.737141:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1788204949.737144:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204949.737147:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204949.737148:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204949.737152:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204949.737153:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204949.737157:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204949.737158:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204949.737163:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00100000:3.0:1788204949.737165:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204949/real 1788204949] req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204949.737169:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204949.737176:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204949.737180:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1788204949.737182:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788204949.737183:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1788204949.737185:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788204949.737186:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.737189:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:3.0:1788204949.737189:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000008:00100000:2.0:1788204949.737193:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:3.0:1788204949.737194:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000008:00000001:2.0:1788204949.737198:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204949.737198:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:2.0:1788204949.737203:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737204:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:2.0:1788204949.737207:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000200:3.0:1788204949.737210:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204949.737211:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204949.737215:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204949.737216:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204949.737220:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204949.737222:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204949.737224:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204949.737228:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000200:3.0:1788204949.737229:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204949.737232:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204949.737240:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204949.737241:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204949.737244:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737249:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204949.737250:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204949.737253:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737254:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204949.737257:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204949.737262:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204949.737265:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204949.737272:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6414) 00000100:00000001:3.0:1788204949.737274:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204949.737280:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737287:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204949.737290:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737292:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737293:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737295:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204949.737297:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204949.737300:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204949.737304:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204949.737306:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204949.737310:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204949.737317:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205913728:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204949.737324:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737326:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.737328:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205913728/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737334:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204949.737336:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204949.737338:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204949.737341:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204949.737343:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204949.737345:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204949.737347:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204949.737349:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:3.0:1788204949.737354:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204949.737355:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204949.737357:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788204949.737360:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204949.737361:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737381:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737389:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204949.737391:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737393:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204949.737408:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204949.737410:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204949.737413:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737418:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204949.737419:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737421:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204949.737422:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204949.737424:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204949.737426:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913856:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204949.737428:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204949.737429:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204949.737430:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737432:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204949.737434:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204949.737436:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.737439:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204949.737441:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204949.737443:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204949.737445:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913856, portal 10 00000100:00000001:3.0:1788204949.737446:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204949.737447:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788204949.737452:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737456:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204949.737458:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204949.737459:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204949.737460:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205913856, offset 0 00000400:00000200:3.0:1788204949.737463:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737468:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204949.737470:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204949.737471:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204949.737473:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204949.737474:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737476:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58995:58995:256:4294967295] 192.168.201.151@tcp LPNI seq info [58975:58975:8:4294967295] 00000400:00000200:3.0:1788204949.737479:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737483:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204949.737486:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737488:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204949.737490:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204949.737492:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204949.737495:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204949.737497:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204949.737499:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737500:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204949.737502:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204949.737505:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204949.737507:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204949.737508:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204949.737509:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204949.737511:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737517:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.737520:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737526:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737528:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204949.737531:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204949.737535:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204949.737538:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204949.737540:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301077147 00000100:00000001:3.0:1788204949.737543:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737545:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301077147 00000100:00000001:3.0:1788204949.737547:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737549:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204949.737551:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737553:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204949.737557:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204949/real 1788204949] req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204949.737565:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204949.737566:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204949.737567:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204949.737569:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204949.737570:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204949.737571:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204949.737574:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204949.737579:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737583:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204949.737583:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737584:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737585:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204949.737587:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737591:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204949.737593:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737598:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204949.737599:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737600:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737601:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737602:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204949.737603:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204949.737604:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204949.737606:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204949.737607:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204949.737609:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204949.737614:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913856:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204949.737617:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204949.737621:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737633:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204949.737634:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737636:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204949.737638:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204949.737640:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204949.737643:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913984:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204949.737646:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204949.737648:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204949.737650:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737651:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204949.737655:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204949.737657:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.737660:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204949.737661:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204949.737664:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204949.737666:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205913984, portal 4 00000100:00000001:3.0:1788204949.737667:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204949.737668:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:3.0:1788204949.737672:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737676:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204949.737678:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204949.737679:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204949.737680:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205913984, offset 0 00000400:00000200:3.0:1788204949.737682:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737685:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737687:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204949.737688:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204949.737690:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204949.737692:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204949.737693:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737695:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58996:58996:256:4294967295] 192.168.201.151@tcp LPNI seq info [58976:58976:8:4294967295] 00000400:00000200:3.0:1788204949.737698:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737702:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204949.737705:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737707:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204949.737711:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204949.737714:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204949.737718:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204949.737720:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204949.737723:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737725:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204949.737728:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204949.737731:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204949.737734:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204949.737736:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204949.737738:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204949.737741:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737749:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.737752:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737760:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737761:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204949.737763:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204949.737766:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204949.737768:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204949.737769:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301077033 00000100:00000001:3.0:1788204949.737771:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737772:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301077033 00000100:00000001:3.0:1788204949.737774:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737775:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204949.737776:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737778:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204949.737780:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204949/real 1788204949] req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204949.737785:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204949.737786:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204949.737787:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204949.737789:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204949.737790:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204949.737791:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204949.737792:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204949.737797:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737800:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204949.737801:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737802:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737803:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204949.737805:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737809:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204949.737811:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737815:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204949.737816:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737817:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737817:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737818:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204949.737820:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204949.737821:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204949.737823:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204949.737825:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204949.737827:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204949.737831:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205913984:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204949.737833:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204949.737835:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737838:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204949.737839:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737840:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204949.737841:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204949.737842:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204949.737844:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914112:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204949.737845:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204949.737846:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204949.737847:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204949.737848:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204949.737850:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204949.737852:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204949.737853:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204949.737855:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204949.737857:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204949.737858:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914112, portal 4 00000100:00000001:3.0:1788204949.737859:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204949.737860:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204949.737864:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737868:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204949.737869:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204949.737870:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204949.737872:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205914112, offset 0 00000400:00000200:3.0:1788204949.737874:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737876:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737878:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204949.737880:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204949.737882:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204949.737883:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204949.737885:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204949.737887:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58997:58997:256:4294967295] 192.168.201.151@tcp LPNI seq info [58977:58977:8:4294967295] 00000400:00000200:3.0:1788204949.737889:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204949.737893:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204949.737896:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737898:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204949.737900:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204949.737902:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204949.737904:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204949.737906:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204949.737908:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204949.737910:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204949.737911:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204949.737913:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204949.737915:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204949.737916:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204949.737918:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204949.737920:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737924:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.737926:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737929:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737930:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204949.737932:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204949.737934:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204949.737936:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204949.737937:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301077039 00000100:00000001:3.0:1788204949.737939:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737940:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301077039 00000100:00000001:3.0:1788204949.737941:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301081089 00000100:00000001:3.0:1788204949.737942:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204949.737943:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.737944:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204949.737946:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204949/real 1788204949] req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204949.737950:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204949.737951:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204949.737952:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204949.737953:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204949.737954:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204949.737955:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204949.737957:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204949.737961:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737965:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204949.737966:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737967:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737968:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204949.737970:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737973:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204949.737975:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.737980:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204949.737981:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737982:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204949.737983:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737984:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204949.737985:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204949.737986:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204949.737988:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204949.737989:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204949.737991:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204949.737995:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914112:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204949.737997:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.737998:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.737999:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205913856/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.738002:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204949.738003:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204949.738004:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204949.738006:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204949.738006:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204949.738007:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204949.738008:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204949.738009:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788204949.738011:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204949.738011:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204949.738012:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204949.738014:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204949.738015:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.738016:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.738018:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205913984/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.738022:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204949.738023:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204949.738023:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204949.738024:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204949.738025:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204949.738026:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204949.738027:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204949.738027:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204949.738028:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204949.738029:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204949.738030:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788204949.738031:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204949.738031:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.738032:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204949.738034:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205914112/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204949.738037:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204949.738037:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204949.738038:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204949.738039:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204949.738040:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204949.738041:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204949.738041:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204949.738042:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788204949.738043:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204949.738044:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204949.738044:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204949.738045:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204949.738046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.738047:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.738049:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204949.738050:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.738051:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204949.738052:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204949.928501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204949.928511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204949.928519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204949.928520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204949.928524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204949.928525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204949.928529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204949.928530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204949.928579:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204949.928584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204949.928587:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204950.248511:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204950.248524:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204950.248530:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204950.248536:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204950.248540:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204950.248546:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.248549:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204950.248552:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.248556:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204950.696463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204950.696472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204950.696474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204950.696479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204950.696483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204950.696486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204950.696488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204950.696491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204950.696494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204950.760497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204950.760497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204950.760505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204950.760505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204950.760508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204950.760508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204950.760514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204950.760514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204950.760518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204950.760519:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204950.760524:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.760524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.760527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204950.760530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204950.760533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204950.760533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204950.760535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204950.760537:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204950.952500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204950.952510:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204950.952513:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204950.952517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952520:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204950.952523:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952526:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952528:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204950.952533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204950.952537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204950.952538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204950.952563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204950.952565:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204950.952567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204951.272451:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204951.272462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204951.272466:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204951.272473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204951.272478:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204951.272483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.272487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204951.272490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.272493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.720435:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204951.720443:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204951.720445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.720450:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204951.720453:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204951.720457:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.720460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.720461:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.720464:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.784487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204951.784487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204951.784495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204951.784495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204951.784498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.784498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.784505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204951.784505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204951.784509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204951.784513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204951.784514:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.784516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.784517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204951.784520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.784523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.784524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.784526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.784529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204951.976508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204951.976517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204951.976524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204951.976525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204951.976528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204951.976531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204951.976535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204951.976536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204951.976601:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204951.976606:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204951.976610:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204952.296462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204952.296468:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204952.296470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204952.296473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204952.296475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204952.296478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204952.296480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204952.296482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204952.296483:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.744484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204952.744495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204952.744497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204952.744505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204952.744509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204952.744514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204952.744517:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204952.744519:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204952.744522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.808511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204952.808516:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204952.808522:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204952.808524:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204952.808527:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204952.808528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204952.808541:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204952.808541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204952.808556:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204952.808556:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204952.808568:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204952.808569:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204952.808577:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204952.808580:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204952.808583:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.808586:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204952.808591:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204952.808594:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204953.000548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204953.000556:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204953.000562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204953.000564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204953.000567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204953.000569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204953.000572:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204953.000573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204953.000595:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204953.000598:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204953.000601:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204953.320455:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204953.320471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204953.320473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204953.320479:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204953.320482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204953.320485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.320488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204953.320491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.320494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.768428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204953.768435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204953.768436:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204953.768440:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204953.768442:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204953.768444:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204953.768446:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204953.768448:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204953.768449:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.832456:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204953.832456:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204953.832462:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204953.832463:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204953.832465:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204953.832466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204953.832471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204953.832471:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204953.832475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204953.832476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204953.832479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.832480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204953.832485:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204953.832487:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204953.832489:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204953.832491:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204953.832494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.832496:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204954.024491:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204954.024494:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204954.024498:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024503:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204954.024505:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024507:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024509:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204954.024514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204954.024519:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204954.024521:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204954.024523:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024524:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204954.024526:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024527:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024529:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204954.024534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204954.024535:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204954.024537:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024538:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204954.024540:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024541:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204954.024543:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204954.024597:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204954.024600:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204954.024603:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204954.344465:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301081089, at 4301081600 00000100:00000001:3.0:1788204954.344466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204954.344471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.344473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204954.344477:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00100000:3.0:1788204954.344478:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204954.344482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204954.344485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.344487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.344489:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.344491:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204954.344558:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301081089, at 4301081600 00000100:00000010:1.0:1788204954.344563:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000001:1.0:1788204954.792483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204954.792495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204954.792497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204954.792501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204954.792503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204954.792506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204954.792508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204954.792509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204954.792511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.856447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204954.856447:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204954.856454:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204954.856455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.856457:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.856458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204954.856463:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204954.856464:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204954.856466:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204954.856469:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204954.856470:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204954.856472:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204954.856473:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.856473:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.856477:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204954.856483:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204954.856483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.856484:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204954.856488:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204954.856488:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204954.856493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204954.856494:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204954.856499:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204954.856508:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6409 00000400:00000200:2.0:1788204954.856517:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204954.856523:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204954.856525:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204954.856529:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856531:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204954.856534:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856536:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204954.856539:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204954.856541:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204954.856543:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204954.856546:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204954.856548:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204954.856551:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204954.856554:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856555:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204954.856557:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856559:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204954.856563:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204954.856566:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856568:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204954.856570:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204954.856573:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204954.856576:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856579:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204954.856581:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204954.856589:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204954.856591:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204954.856594:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204954.856595:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856597:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204954.856599:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204954.856609:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204954.856617:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204954.856622:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204954.856628:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204954.856635:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204954.856638:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204954.856641:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204954.856656:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204954.856658:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204954.856660:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204954.856661:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204954.856662:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204954.856664:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204954.856665:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6409 00000400:00000200:2.0:1788204954.856668:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204954.856671:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204954.856673:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204954.856675:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856677:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204954.856677:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204954.856681:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788204954.856682:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856685:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204954.856685:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204954.856688:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204954.856689:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204954.856692:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:3.0:1788204954.856692:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204954.856696:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204954.856700:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204954.856705:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204954.856706:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.856709:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204954.856711:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204954.856712:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204954.856714:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788204954.856715:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204954.856718:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204954.856719:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856721:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788204954.856724:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205914240:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204954.856728:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1788204954.856730:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204954.856731:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788204954.856734:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204954.856735:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.856739:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204954.856740:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204954.856742:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1788204954.856745:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:2.0:1788204954.856746:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204954.856751:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.856751:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856754:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204954.856758:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204954.856761:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:2.0:1788204954.856762:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856766:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204954.856768:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000010:2.0:1788204954.856769:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204954.856773:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788204954.856775:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204954.856776:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856777:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204954.856777:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788204954.856778:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 02000000:00000001:2.0:1788204954.856779:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204954.856785:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788204954.856787:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204954.856787:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914240, portal 25 00000100:00000001:3.0:1788204954.856793:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1788204954.856796:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204954.856796:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788204954.856804:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204954.856805:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204954.856809:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.856815:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204954.856816:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204954.856821:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204954.856825:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000001:2.0:1788204954.856828:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204954.856829:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205914240, offset 0 00000100:00080000:2.0:1788204954.856831:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204954.856834:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204954.856837:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204954.856837:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204954.856840:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204954.856842:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204954.856845:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6409 00000400:00000200:3.0:1788204954.856847:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.856853:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204954.856857:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204954.856860:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204954.856863:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204954.856865:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204954.856869:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58998:58998:256:4294967295] 192.168.201.151@tcp LPNI seq info [58978:58978:8:4294967295] 00000400:00000200:3.0:1788204954.856873:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204954.856878:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204954.856883:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204954.856886:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204954.856891:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204954.856891:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788204954.856894:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204954.856898:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856900:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204954.856902:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204954.856904:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000800:00000200:3.0:1788204954.856905:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788204954.856906:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204954.856908:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204954.856910:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204954.856911:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:3.0:1788204954.856912:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000040:2.0:1788204954.856914:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856917:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204954.856918:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.856921:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788204954.856922:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:2.0:1788204954.856924:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.0:1788204954.856925:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1788204954.856930:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.1:1788204954.856933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788204954.856935:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204954.856937:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 02000000:00000001:2.0:1788204954.856940:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:3.0:1788204954.856943:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788204954.856945:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:3.0:1788204954.856945:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 02000000:00000001:2.0:1788204954.856950:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204954.856951:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204954.856956:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204954.856958:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000008:00100000:2.0:1788204954.856960:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204954.856962:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000008:00000001:2.0:1788204954.856965:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204954.856966:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204954.856970:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204954.856971:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204954.856975:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204954.856976:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204954.856980:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204954.856985:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204954.856987:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.856990:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204954.856992:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204954.856995:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.856998:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857001:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857005:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204954.857006:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:3.0:1788204954.857006:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204954.857011:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857012:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204954.857016:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204954.857017:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204954.857021:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301086209 00000100:00000001:2.0:1788204954.857027:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000010:3.0:1788204954.857027:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4720. 00000100:00000001:2.0:1788204954.857031:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204954.857034:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857035:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788204954.857042:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857042:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301086209 00000100:00000010:3.0:1788204954.857046:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4420. 00000100:00080000:2.0:1788204954.857050:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204954.857051:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788204954.857056:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204954.857056:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204954.857060:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204954.857060:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204954.857062:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204954.857064:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204954.857065:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1788204954.857069:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204954/real 1788204954] req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204954.857070:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6409 00000400:00000200:2.0:1788204954.857078:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204954.857083:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000001:3.0:1788204954.857085:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788204954.857086:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000001:3.0:1788204954.857087:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1788204954.857090:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204954.857090:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000100:00000001:2.0:1788204954.857094:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204954.857095:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000040:2.0:1788204954.857098:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204954.857098:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204954.857101:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788204954.857102:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204954.857106:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000200:3.0:1788204954.857106:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204954.857109:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204954.857112:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204954.857115:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204954.857119:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000200:3.0:1788204954.857119:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204954.857123:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204954.857128:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204954.857132:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204954.857135:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204954.857138:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204954.857139:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204954.857143:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204954.857143:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204954.857145:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204954.857148:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204954.857149:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204954.857152:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204954.857153:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204954.857154:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204954.857158:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.857159:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204954.857161:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1788204954.857162:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788204954.857163:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204954.857166:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204954.857170:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204954.857173:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204954.857177:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204954.857179:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857179:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204954.857184:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204954.857185:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204954.857187:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204954.857189:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204954.857190:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788204954.857194:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204954.857200:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204954.857207:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204954.857212:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204954.857213:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204954.857216:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857217:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1788204954.857221:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857221:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857225:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204954.857229:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788204954.857230:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204954.857234:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6419) 00000100:00080000:3.0:1788204954.857235:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204954.857241:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204954.857245:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204954.857250:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204954.857256:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205914240:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204954.857259:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857260:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857261:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205914240/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857265:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204954.857267:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857269:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204954.857271:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204954.857272:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204954.857274:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204954.857275:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204954.857277:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:3.0:1788204954.857279:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204954.857280:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204954.857281:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788204954.857283:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204954.857284:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857285:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857289:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.857290:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857291:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204954.857292:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204954.857294:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204954.857296:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857300:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.857301:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857302:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204954.857303:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204954.857304:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204954.857328:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914368:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204954.857330:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204954.857332:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204954.857333:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857334:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857336:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204954.857337:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204954.857340:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204954.857342:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204954.857344:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204954.857346:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914368, portal 10 00000100:00000001:3.0:1788204954.857347:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204954.857348:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788204954.857352:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857357:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204954.857358:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204954.857359:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204954.857361:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205914368, offset 0 00000400:00000200:3.0:1788204954.857378:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857381:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857383:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204954.857384:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204954.857386:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204954.857388:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204954.857389:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857391:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [58999:58999:256:4294967295] 192.168.201.151@tcp LPNI seq info [58979:58979:8:4294967295] 00000400:00000200:3.0:1788204954.857394:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857398:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204954.857400:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857403:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204954.857405:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204954.857407:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204954.857409:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204954.857411:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204954.857413:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857414:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204954.857416:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204954.857418:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204954.857420:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204954.857421:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204954.857423:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204954.857425:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857430:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857431:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857439:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857440:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204954.857442:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204954.857444:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204954.857447:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204954.857448:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301082267 00000100:00000001:3.0:1788204954.857449:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857451:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301082267 00000100:00000001:3.0:1788204954.857452:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857453:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204954.857454:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857455:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204954.857457:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204954/real 1788204954] req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204954.857462:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204954.857463:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204954.857463:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204954.857465:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204954.857466:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204954.857467:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204954.857469:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204954.857473:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857476:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204954.857477:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857478:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857479:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204954.857482:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857494:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204954.857498:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857504:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204954.857507:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857508:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857509:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857511:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204954.857513:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204954.857516:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204954.857519:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204954.857521:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204954.857525:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204954.857531:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914368:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204954.857534:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204954.857536:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857540:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.857541:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857542:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204954.857543:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204954.857544:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204954.857546:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914496:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204954.857548:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204954.857549:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204954.857550:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857551:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857553:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204954.857555:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204954.857557:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204954.857558:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204954.857561:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204954.857562:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914496, portal 4 00000100:00000001:3.0:1788204954.857564:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204954.857564:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:3.0:1788204954.857568:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857572:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204954.857573:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204954.857574:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204954.857575:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205914496, offset 0 00000400:00000200:3.0:1788204954.857578:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857580:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857582:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204954.857583:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204954.857585:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204954.857587:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204954.857588:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857590:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59000:59000:256:4294967295] 192.168.201.151@tcp LPNI seq info [58980:58980:8:4294967295] 00000400:00000200:3.0:1788204954.857593:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857597:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204954.857599:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857601:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204954.857604:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204954.857606:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204954.857610:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204954.857612:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204954.857616:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857618:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204954.857621:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204954.857624:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204954.857627:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204954.857628:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204954.857630:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204954.857634:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857641:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857645:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857651:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857653:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204954.857656:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204954.857658:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204954.857661:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204954.857663:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301082153 00000100:00000001:3.0:1788204954.857665:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857667:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301082153 00000100:00000001:3.0:1788204954.857669:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857671:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204954.857673:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857675:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204954.857679:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204954/real 1788204954] req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204954.857687:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204954.857688:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204954.857689:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204954.857691:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204954.857692:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204954.857694:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204954.857696:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204954.857700:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857704:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204954.857705:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857706:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857707:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204954.857709:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857712:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204954.857715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857719:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204954.857720:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857721:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857721:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857722:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204954.857724:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204954.857725:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204954.857727:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204954.857728:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204954.857730:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204954.857734:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914496:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204954.857736:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204954.857738:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857743:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204954.857744:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857746:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204954.857748:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204954.857750:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204954.857753:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914624:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204954.857755:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204954.857757:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204954.857758:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204954.857760:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857763:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204954.857765:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204954.857768:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204954.857770:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204954.857774:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204954.857776:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914624, portal 4 00000100:00000001:3.0:1788204954.857778:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204954.857780:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204954.857785:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857791:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204954.857792:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204954.857793:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204954.857795:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205914624, offset 0 00000400:00000200:3.0:1788204954.857797:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857800:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857802:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204954.857804:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204954.857805:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204954.857807:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204954.857808:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204954.857810:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59001:59001:256:4294967295] 192.168.201.151@tcp LPNI seq info [58981:58981:8:4294967295] 00000400:00000200:3.0:1788204954.857812:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204954.857816:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204954.857819:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857821:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204954.857824:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204954.857825:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204954.857828:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204954.857829:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788204954.857832:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204954.857833:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204954.857835:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204954.857837:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204954.857838:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204954.857839:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204954.857841:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204954.857843:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857847:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857849:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857853:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857854:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204954.857856:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204954.857857:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204954.857859:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204954.857860:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301082159 00000100:00000001:3.0:1788204954.857862:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857863:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301082159 00000100:00000001:3.0:1788204954.857865:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301086209 00000100:00000001:3.0:1788204954.857867:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204954.857869:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857871:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204954.857874:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204954/real 1788204954] req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204954.857881:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204954.857883:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204954.857884:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204954.857886:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204954.857888:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204954.857890:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204954.857893:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204954.857901:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857907:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204954.857909:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857910:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857911:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204954.857913:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857917:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204954.857920:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857923:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204954.857925:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857926:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204954.857927:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857928:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204954.857929:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204954.857930:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204954.857932:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204954.857933:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204954.857936:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204954.857940:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914624:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204954.857942:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857943:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857944:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205914368/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857948:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204954.857949:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857950:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204954.857951:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204954.857952:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204954.857953:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204954.857954:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204954.857955:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788204954.857957:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204954.857958:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204954.857959:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788204954.857960:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204954.857961:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857962:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857964:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205914496/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857967:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204954.857968:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857969:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204954.857970:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204954.857971:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204954.857971:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204954.857972:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204954.857973:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204954.857975:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204954.857976:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204954.857976:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788204954.857978:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204954.857978:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857979:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204954.857981:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205914624/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204954.857984:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204954.857984:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204954.857985:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204954.857986:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204954.857987:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204954.857987:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204954.857988:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204954.857989:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788204954.857990:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204954.857990:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204954.857991:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204954.857992:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204954.857993:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857993:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204954.857996:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.857996:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204954.857997:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204954.857998:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204955.048515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204955.048520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204955.048524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204955.048525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204955.048527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204955.048528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204955.048529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204955.048530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204955.048543:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204955.048545:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204955.048547:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204955.368441:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204955.368446:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204955.368448:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204955.368452:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204955.368454:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204955.368457:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.368459:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204955.368461:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.368463:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.816471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204955.816477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204955.816478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204955.816482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204955.816484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204955.816487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204955.816488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204955.816490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204955.816492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.880466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204955.880468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204955.880473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204955.880474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204955.880475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.880478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204955.880480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204955.880483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204955.880483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204955.880487:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204955.880488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204955.880490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204955.880492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.880493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.880499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.880500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204955.880503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204955.880506:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204956.072483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204956.072490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204956.072496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204956.072498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204956.072502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204956.072504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204956.072507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204956.072508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204956.072566:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204956.072569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204956.072572:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204956.392467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204956.392475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204956.392477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204956.392482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204956.392486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204956.392489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.392493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204956.392495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.392498:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.840741:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204956.840747:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204956.840748:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.840760:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204956.840772:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204956.840775:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.840778:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.840781:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.840784:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.904419:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204956.904425:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204956.904426:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204956.904427:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.904431:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204956.904432:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204956.904434:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.904436:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204956.904440:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204956.904440:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788204956.904444:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.904445:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.904448:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.904452:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204956.904456:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.904457:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204956.904458:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.904460:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204957.096509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204957.096513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204957.096516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204957.096517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204957.096519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204957.096520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204957.096522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204957.096523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204957.096539:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204957.096542:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204957.096545:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204957.416457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204957.416463:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204957.416464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204957.416468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204957.416471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204957.416473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.416475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204957.416477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.416479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.864480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204957.864487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204957.864490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204957.864495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204957.864498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204957.864502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204957.864505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204957.864507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204957.864510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.928481:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204957.928481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204957.928489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204957.928490:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204957.928492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.928494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204957.928500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204957.928500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204957.928506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204957.928507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204957.928512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.928512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.928519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204957.928523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.928526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.928527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204957.928529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204957.928532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204958.120464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204958.120469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204958.120473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204958.120474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204958.120477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204958.120478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204958.120480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204958.120481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204958.120549:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204958.120553:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204958.120556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204958.440461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204958.440467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204958.440469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204958.440474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204958.440478:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204958.440482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204958.440484:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204958.440487:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204958.440490:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204958.888499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204958.888507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204958.888509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204958.888514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204958.888517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204958.888521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204958.888523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204958.888526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204958.888528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204958.952440:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204958.952447:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204958.952449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204958.952449:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204958.952451:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204958.952452:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204958.952454:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:1.0:1788204958.952454:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788204958.952457:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204958.952458:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204958.952461:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204958.952461:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204958.952464:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204958.952467:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204958.952470:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204958.952470:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204958.952471:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204958.952473:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204959.144505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204959.144511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204959.144517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204959.144518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204959.144520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204959.144522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204959.144525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204959.144526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204959.144544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204959.144546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204959.144549:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204959.464469:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301086209, at 4301086720 00000100:00000040:1.0:1788204959.464469:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301086209, at 4301086720 00000100:00000001:3.0:1788204959.464469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204959.464476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204959.464478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204959.464483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204959.464484:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000001:3.0:1788204959.464486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204959.464487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000001:3.0:1788204959.464489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.464493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204959.464496:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.464499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.912460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204959.912467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204959.912470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204959.912475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204959.912479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204959.912486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204959.912489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204959.912491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204959.912494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204959.976464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788204959.976464:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204959.976468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204959.976471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788204959.976473:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204959.976474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204959.976474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204959.976478:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204959.976478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204959.976481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788204959.976482:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204959.976484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204959.976488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204959.976488:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204959.976489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204959.976494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204959.976494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204959.976495:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6414 00000100:00000001:0.0:1788204959.976501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204959.976505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204959.976505:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204959.976511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204959.976511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1788204959.976513:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204959.976515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788204959.976517:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204959.976520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204959.976521:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976526:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976528:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976530:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204959.976533:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204959.976534:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788204959.976536:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204959.976538:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788204959.976540:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788204959.976542:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788204959.976544:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976546:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976547:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976549:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204959.976551:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788204959.976553:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976554:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204959.976556:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204959.976558:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204959.976560:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976561:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204959.976562:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204959.976569:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788204959.976571:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788204959.976574:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204959.976575:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976577:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204959.976578:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204959.976581:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204959.976586:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788204959.976591:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204959.976597:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204959.976604:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204959.976607:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204959.976610:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204959.976624:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204959.976626:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204959.976628:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204959.976629:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204959.976630:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204959.976633:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204959.976634:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6414 00000400:00000200:1.0:1788204959.976636:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204959.976637:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204959.976639:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204959.976640:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976642:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976643:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976644:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204959.976646:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204959.976646:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788204959.976647:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204959.976649:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788204959.976650:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788204959.976651:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788204959.976653:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976654:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976655:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976657:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204959.976658:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788204959.976660:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976661:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204959.976663:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204959.976667:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204959.976669:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.976669:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204959.976671:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204959.976674:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:1.0:1788204959.976675:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204959.976678:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:1.0:1788204959.976679:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976682:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204959.976684:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788204959.976687:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788204959.976690:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:3.0:1788204959.976690:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204959.976692:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976695:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204959.976697:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204959.976699:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204959.976704:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:3.0:1788204959.976707:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204959.976708:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.976711:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204959.976714:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204959.976717:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204959.976721:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204959.976722:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204959.976725:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.976727:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204959.976729:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204959.976735:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205914752:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788204959.976737:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204959.976741:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204959.976742:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1788204959.976744:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204959.976746:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204959.976747:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788204959.976748:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788204959.976749:0:12771:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1788204959.976752:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204959.976753:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6414 02000000:00000001:3.0:1788204959.976758:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1788204959.976759:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204959.976762:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204959.976763:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204959.976765:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000010:3.0:1788204959.976766:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000001:1.0:1788204959.976767:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976770:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204959.976772:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976773:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204959.976776:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204959.976777:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788204959.976778:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788204959.976780:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204959.976781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788204959.976782:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788204959.976785:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788204959.976788:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204959.976788:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:1.0:1788204959.976789:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976792:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976794:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788204959.976796:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788204959.976797:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:3.0:1788204959.976798:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:1.0:1788204959.976799:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204959.976802:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204959.976804:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204959.976805:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976807:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204959.976808:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000200:3.0:1788204959.976809:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914752, portal 25 00000008:00000001:1.0:1788204959.976810:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976814:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.976815:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:1.0:1788204959.976817:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000100:00000001:3.0:1788204959.976818:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000020:00000040:1.0:1788204959.976820:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788204959.976823:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204959.976824:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976825:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204959.976826:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204959.976828:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204959.976830:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788204959.976831:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788204959.976834:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204959.976841:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204959.976844:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204959.976851:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788204959.976851:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976854:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204959.976857:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000040:1.0:1788204959.976858:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204959.976863:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205914752, offset 0 00000100:00000001:1.0:1788204959.976865:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204959.976868:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204959.976870:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204959.976871:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204959.976873:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204959.976875:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204959.976875:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788204959.976879:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6414 00000400:00000200:1.0:1788204959.976881:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204959.976883:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204959.976884:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204959.976887:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204959.976889:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788204959.976890:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976893:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976895:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788204959.976896:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788204959.976897:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788204959.976899:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204959.976900:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:1.0:1788204959.976901:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788204959.976905:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788204959.976907:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788204959.976910:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204959.976910:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204959.976914:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204959.976915:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976917:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204959.976918:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:1.0:1788204959.976919:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788204959.976924:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976927:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.976927:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1788204959.976932:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204959.976934:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204959.976934:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:1.0:1788204959.976938:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976939:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204959.976942:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:3.0:1788204959.976943:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59002:59002:256:4294967295] 192.168.201.151@tcp LPNI seq info [58982:58982:8:4294967295] 00000008:00000001:1.0:1788204959.976944:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976948:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204959.976953:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000400:00000200:3.0:1788204959.976954:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788204959.976956:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788204959.976959:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204959.976959:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.976960:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204959.976961:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204959.976963:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204959.976966:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788204959.976969:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.976971:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00100000:1.0:1788204959.976974:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204959.976982:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:3.0:1788204959.976983:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788204959.976984:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204959.976989:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1788204959.976991:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000001:1.0:1788204959.976995:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204959.976999:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6424) 00000800:00000200:3.0:1788204959.977003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204959.977007:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204959.977013:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204959.977017:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204959.977021:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977024:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204959.977031:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204959.977035:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204959.977039:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204959.977041:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204959.977045:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204959.977049:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977059:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.977062:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977069:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977070:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204959.977076:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.977079:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204959.977082:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301091329 00000100:00000010:3.0:1788204959.977087:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c60. 00000100:00000001:3.0:1788204959.977093:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204959.977099:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301091329 00000100:00000010:3.0:1788204959.977102:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4780. 00000100:00000001:3.0:1788204959.977105:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204959.977108:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204959.977110:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977112:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204959.977117:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204959/real 1788204959] req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204959.977125:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204959.977126:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204959.977128:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204959.977131:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204959.977132:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204959.977134:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204959.977138:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204959.977146:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977152:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204959.977154:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977156:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977158:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204959.977163:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977170:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204959.977174:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977180:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204959.977182:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977184:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977185:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977187:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204959.977189:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204959.977192:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204959.977195:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204959.977197:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204959.977201:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204959.977208:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205914752:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204959.977212:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977214:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.977234:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205914752/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977246:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204959.977249:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204959.977251:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204959.977255:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204959.977257:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204959.977258:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204959.977260:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204959.977263:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc400. 02000000:00000001:3.0:1788204959.977266:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204959.977268:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204959.977270:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:3.0:1788204959.977273:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204959.977274:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977276:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977282:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204959.977283:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977285:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204959.977286:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204959.977288:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204959.977291:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977298:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204959.977299:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977301:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204959.977302:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204959.977304:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204959.977307:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914880:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204959.977310:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204959.977312:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204959.977313:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977315:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204959.977318:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204959.977320:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.977323:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204959.977326:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204959.977329:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204959.977331:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205914880, portal 10 00000100:00000001:3.0:1788204959.977333:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204959.977334:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:3.0:1788204959.977340:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977346:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204959.977348:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204959.977350:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204959.977352:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205914880, offset 0 00000400:00000200:3.0:1788204959.977356:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977360:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977387:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204959.977390:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204959.977393:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204959.977395:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204959.977397:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977400:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59003:59003:256:4294967295] 192.168.201.151@tcp LPNI seq info [58983:58983:8:4294967295] 00000400:00000200:3.0:1788204959.977405:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977411:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204959.977416:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977419:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204959.977423:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204959.977425:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204959.977429:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204959.977432:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204959.977435:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977437:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204959.977440:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204959.977443:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204959.977446:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204959.977448:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204959.977451:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204959.977455:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977462:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.977465:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977472:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977473:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204959.977477:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204959.977480:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204959.977483:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204959.977485:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301087387 00000100:00000001:3.0:1788204959.977488:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.977490:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301087387 00000100:00000001:3.0:1788204959.977492:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.977494:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204959.977496:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977498:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204959.977501:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204959/real 1788204959] req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204959.977508:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204959.977509:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204959.977511:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204959.977513:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204959.977515:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204959.977517:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204959.977520:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204959.977528:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977547:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204959.977549:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977551:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977552:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204959.977556:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977562:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204959.977566:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977573:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204959.977575:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977577:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977578:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977580:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204959.977582:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204959.977584:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204959.977587:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204959.977590:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204959.977593:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204959.977600:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205914880:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204959.977604:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204959.977607:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977613:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204959.977614:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977617:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204959.977618:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204959.977620:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204959.977623:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915008:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204959.977626:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204959.977628:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204959.977629:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977631:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204959.977634:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204959.977636:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.977639:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204959.977642:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204959.977645:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204959.977647:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915008, portal 4 00000100:00000001:3.0:1788204959.977649:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204959.977651:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:3.0:1788204959.977656:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977662:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204959.977664:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204959.977667:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204959.977670:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205915008, offset 0 00000400:00000200:3.0:1788204959.977673:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977676:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977680:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204959.977682:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204959.977684:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204959.977687:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204959.977689:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977692:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59004:59004:256:4294967295] 192.168.201.151@tcp LPNI seq info [58984:58984:8:4294967295] 00000400:00000200:3.0:1788204959.977696:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977701:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204959.977706:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977709:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204959.977713:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204959.977715:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204959.977719:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204959.977722:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204959.977725:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977727:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204959.977729:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204959.977732:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204959.977735:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204959.977737:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204959.977739:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204959.977742:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977749:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.977752:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977757:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977759:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204959.977762:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204959.977765:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204959.977768:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204959.977769:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301087273 00000100:00000001:3.0:1788204959.977772:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.977774:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301087273 00000100:00000001:3.0:1788204959.977776:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.977777:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204959.977779:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977781:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204959.977784:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204959/real 1788204959] req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204959.977791:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204959.977792:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204959.977794:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204959.977796:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204959.977798:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204959.977800:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204959.977803:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204959.977811:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977817:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204959.977818:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977819:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977821:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204959.977824:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977829:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204959.977833:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977839:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204959.977841:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977842:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.977843:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.977845:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204959.977847:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204959.977850:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204959.977852:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204959.977854:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204959.977859:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204959.977865:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915008:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204959.977869:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204959.977871:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977877:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204959.977878:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977881:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204959.977882:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204959.977884:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204959.977887:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915136:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204959.977889:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204959.977891:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204959.977892:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204959.977894:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204959.977897:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204959.977899:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204959.977901:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204959.977904:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204959.977907:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204959.977909:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915136, portal 4 00000100:00000001:3.0:1788204959.977911:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204959.977912:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:3.0:1788204959.977917:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977921:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204959.977922:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204959.977924:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204959.977925:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205915136, offset 0 00000400:00000200:3.0:1788204959.977928:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977930:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977932:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204959.977934:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204959.977936:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204959.977937:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204959.977939:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204959.977941:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59005:59005:256:4294967295] 192.168.201.151@tcp LPNI seq info [58985:58985:8:4294967295] 00000400:00000200:3.0:1788204959.977943:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204959.977946:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204959.977949:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977951:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204959.977953:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204959.977954:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204959.977957:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204959.977959:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204959.977971:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204959.977972:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204959.977974:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204959.977976:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204959.977979:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204959.977980:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204959.977981:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204959.977983:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977988:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.977989:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.977993:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.977994:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204959.977996:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204959.977998:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204959.978000:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204959.978002:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301087279 00000100:00000001:3.0:1788204959.978003:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.978004:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301087279 00000100:00000001:3.0:1788204959.978006:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301091329 00000100:00000001:3.0:1788204959.978007:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204959.978008:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.978009:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204959.978011:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204959/real 1788204959] req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204959.978015:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204959.978016:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204959.978017:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204959.978018:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204959.978019:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204959.978020:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204959.978022:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204959.978026:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978029:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204959.978030:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978031:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.978032:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204959.978034:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978037:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204959.978039:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978042:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204959.978044:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978045:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204959.978046:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978047:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204959.978048:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204959.978050:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204959.978051:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204959.978052:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204959.978054:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204959.978058:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915136:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204959.978060:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978061:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.978063:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205914880/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978067:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204959.978068:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204959.978069:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204959.978070:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204959.978071:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204959.978072:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204959.978073:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204959.978074:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:3.0:1788204959.978076:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204959.978076:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204959.978077:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:3.0:1788204959.978079:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204959.978080:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978081:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.978082:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205915008/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978090:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204959.978090:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204959.978091:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204959.978092:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204959.978093:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204959.978094:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204959.978095:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204959.978095:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:3.0:1788204959.978096:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204959.978097:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204959.978098:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:3.0:1788204959.978099:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204959.978100:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978100:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204959.978102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205915136/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204959.978105:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204959.978105:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204959.978106:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204959.978107:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204959.978108:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204959.978109:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204959.978109:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204959.978110:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:3.0:1788204959.978111:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204959.978112:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204959.978112:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:3.0:1788204959.978113:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204959.978114:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978115:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.978117:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204959.978118:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.978119:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204959.978120:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204960.168483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204960.168492:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204960.168494:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204960.168497:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168500:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204960.168502:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168504:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168506:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204960.168511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204960.168514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204960.168516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204960.168580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204960.168584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204960.168587:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204960.488500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204960.488508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204960.488510:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204960.488516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204960.488519:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204960.488523:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204960.488526:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204960.488536:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204960.488540:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204960.937030:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204960.937036:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204960.937037:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204960.937040:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204960.937043:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204960.937046:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204960.937048:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204960.937049:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204960.937051:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204961.000468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204961.000475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204961.000476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204961.000477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204961.000481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788204961.000484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204961.000484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204961.000490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788204961.000490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204961.000495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204961.000495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204961.000498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204961.000499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204961.000503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204961.000506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204961.000509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204961.000512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204961.000515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204961.192526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204961.192532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204961.192538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204961.192539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204961.192542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204961.192543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204961.192546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204961.192547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204961.192563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204961.192566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204961.192568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204961.512475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204961.512481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204961.512483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204961.512488:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204961.512491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204961.512495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204961.512498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204961.512501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204961.512503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.960477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204961.960484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204961.960486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.960491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204961.960494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204961.960499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.960501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204961.960504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.960507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204962.024474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204962.024481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204962.024483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204962.024488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204962.024491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204962.024495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204962.024497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204962.024499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204962.024501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204962.024886:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204962.024889:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204962.024890:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204962.024893:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204962.024895:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204962.024897:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204962.024899:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204962.024901:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204962.024903:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204962.216471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204962.216476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204962.216480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204962.216481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204962.216483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204962.216484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204962.216487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204962.216488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204962.216554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204962.216558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204962.216561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204962.536499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204962.536506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204962.536508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204962.536513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204962.536517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204962.536521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204962.536523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204962.536526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204962.536529:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204962.984500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204962.984507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204962.984509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204962.984515:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204962.984518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204962.984522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204962.984524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204962.984526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204962.984529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204963.048467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204963.048467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204963.048474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204963.048475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204963.048476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204963.048478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204963.048481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204963.048484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204963.048485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204963.048489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204963.048490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204963.048494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204963.048494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204963.048497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204963.048500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204963.048500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204963.048504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204963.048507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204963.240568:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204963.240571:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204963.240574:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240578:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204963.240579:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240581:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240582:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204963.240586:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204963.240590:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204963.240590:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204963.240591:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240592:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204963.240593:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240594:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240595:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204963.240598:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204963.240599:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204963.240601:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240602:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204963.240603:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240605:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204963.240606:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204963.240627:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204963.240629:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204963.240632:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204963.560471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204963.560482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204963.560484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204963.560495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204963.560500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204963.560505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204963.560509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204963.560512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204963.560515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204964.008500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204964.008507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204964.008509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204964.008514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204964.008517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204964.008521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204964.008523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204964.008525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204964.008528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204964.072502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204964.072502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204964.072510:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204964.072510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204964.072513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204964.072514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204964.072522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788204964.072522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204964.072527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204964.072528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204964.072532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204964.072532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204964.072536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204964.072540:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204964.072542:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204964.072544:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204964.072547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204964.072551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204964.264483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204964.264489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204964.264496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204964.264497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204964.264501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204964.264502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204964.264504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204964.264505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204964.264563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204964.264566:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204964.264569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204964.584473:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301091329, at 4301091840 00000100:00000040:1.0:1788204964.584473:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301091329, at 4301091840 00000100:00000010:0.0:1788204964.584487:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000010:1.0:1788204964.584490:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00000001:3.0:1788204964.584575:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204964.584579:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204964.584580:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204964.584584:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204964.584587:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204964.584589:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204964.584591:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204964.584594:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204964.584596:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204965.032516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204965.032523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204965.032525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204965.032531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204965.032535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204965.032539:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204965.032542:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204965.032545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204965.032548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204965.096475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788204965.096480:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788204965.096481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204965.096484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204965.096489:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1788204965.096489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204965.096494:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204965.096494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204965.096498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204965.096499:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204965.096500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204965.096503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204965.096505:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204965.096506:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204965.096515:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6419 00000400:00000200:0.0:1788204965.096525:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204965.096531:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204965.096534:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204965.096534:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204965.096537:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788204965.096539:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204965.096540:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096543:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788204965.096544:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000040:0.0:1788204965.096548:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204965.096548:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204965.096551:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788204965.096551:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204965.096556:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788204965.096556:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204965.096557:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788204965.096558:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:2.0:1788204965.096559:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204965.096563:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204965.096566:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204965.096567:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204965.096569:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204965.096572:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096573:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204965.096574:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096575:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204965.096577:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204965.096580:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096581:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204965.096584:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204965.096586:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204965.096588:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096589:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204965.096591:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204965.096598:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204965.096600:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204965.096603:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204965.096604:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096606:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204965.096608:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204965.096611:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204965.096617:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788204965.096622:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204965.096628:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204965.096634:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204965.096645:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204965.096651:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204965.096666:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204965.096669:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.096672:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204965.096673:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204965.096675:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204965.096677:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204965.096681:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6419 00000400:00000200:0.0:1788204965.096684:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204965.096687:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204965.096689:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204965.096692:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096695:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204965.096697:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096699:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204965.096701:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204965.096702:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788204965.096703:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204965.096704:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204965.096706:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204965.096706:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204965.096707:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204965.096708:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788204965.096710:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788204965.096712:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096714:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204965.096714:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204965.096716:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096718:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204965.096720:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204965.096722:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204965.096722:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.096723:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096725:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.096726:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204965.096727:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204965.096728:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204965.096731:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204965.096731:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204965.096733:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096734:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1788204965.096734:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205915264:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204965.096737:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788204965.096738:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204965.096741:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.096741:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788204965.096742:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.096744:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204965.096745:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 02000000:00000001:3.0:1788204965.096746:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1788204965.096747:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788204965.096749:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000010:3.0:1788204965.096750:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:0.0:1788204965.096751:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096752:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204965.096752:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204965.096753:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204965.096755:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204965.096756:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:0.0:1788204965.096758:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000400:00000010:3.0:1788204965.096760:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204965.096761:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204965.096765:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204965.096766:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000001:0.0:1788204965.096771:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204965.096772:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915264, portal 25 00000100:00000001:0.0:1788204965.096773:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.096774:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204965.096775:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:0.0:1788204965.096777:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204965.096781:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204965.096784:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204965.096787:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204965.096788:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788204965.096790:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:3.0:1788204965.096790:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096792:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204965.096793:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788204965.096793:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000001:0.0:1788204965.096796:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788204965.096796:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205915264, offset 0 00000100:00080000:0.0:1788204965.096798:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6419 00000400:00000200:0.0:1788204965.096800:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.096801:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788204965.096803:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204965.096805:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204965.096807:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204965.096808:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204965.096809:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204965.096811:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096812:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204965.096814:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788204965.096814:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:0.0:1788204965.096816:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788204965.096817:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204965.096819:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204965.096819:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1788204965.096821:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204965.096822:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204965.096823:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:0.0:1788204965.096825:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096827:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204965.096827:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204965.096829:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204965.096829:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096831:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204965.096833:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204965.096833:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59006:59006:256:4294967295] 192.168.201.151@tcp LPNI seq info [58986:58986:8:4294967295] 00000100:00080000:0.0:1788204965.096835:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788204965.096837:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204965.096838:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204965.096839:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204965.096841:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204965.096843:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096844:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204965.096846:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788204965.096848:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096850:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204965.096853:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000400:00000200:3.0:1788204965.096853:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:0.0:1788204965.096855:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204965.096857:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204965.096858:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096859:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204965.096861:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204965.096862:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204965.096862:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000010:0.0:1788204965.096865:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000800:00000010:3.0:1788204965.096867:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00000001:0.0:1788204965.096868:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204965.096871:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788204965.096872:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204965.096876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000100:00000001:0.0:1788204965.096878:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204965.096880:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204965.096880:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204965.096883:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204965.096884:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788204965.096888:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096890:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204965.096890:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000100:00080000:0.0:1788204965.096893:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.096895:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788204965.096896:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204965.096897:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204965.096898:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204965.096900:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788204965.096901:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204965.096903:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6419 00000400:00000200:3.0:1788204965.096904:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000200:0.0:1788204965.096906:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204965.096906:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000400:00000200:0.0:1788204965.096908:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204965.096910:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204965.096910:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1788204965.096912:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204965.096913:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204965.096914:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204965.096916:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096919:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204965.096921:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204965.096922:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788204965.096922:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788204965.096923:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204965.096925:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000040:3.0:1788204965.096925:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204965.096927:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204965.096929:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204965.096932:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204965.096932:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096933:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204965.096934:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1788204965.096935:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096936:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204965.096938:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204965.096939:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204965.096941:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204965.096943:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204965.096943:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204965.096944:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204965.096946:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301096449 02000000:00000001:0.0:1788204965.096947:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204965.096948:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096950:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000010:3.0:1788204965.096950:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4ae0. 00000008:00100000:0.0:1788204965.096952:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204965.096954:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.096955:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204965.096960:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301096449 00000100:00000001:0.0:1788204965.096963:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204965.096963:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f44e0. 00000100:00000010:0.0:1788204965.096966:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204965.096966:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1788204965.096968:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204965.096969:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204965.096970:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204965.096971:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.096971:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204965.096972:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204965.096973:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788204965.096974:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204965.096975:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204965.096977:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788204965.096978:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204965.096981:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204965.096985:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788204965.096986:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204965.096988:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204965.096990:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204965.096991:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204965.096992:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000001:0.0:1788204965.096993:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204965.096994:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000040:0.0:1788204965.096996:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.096996:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204965.096999:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204965.097003:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204965.097007:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6430) 00000100:00000200:3.0:1788204965.097007:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097013:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204965.097015:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097017:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097019:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204965.097023:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097030:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204965.097033:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097044:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204965.097046:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097047:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097048:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097049:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204965.097051:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204965.097052:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.097055:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204965.097056:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204965.097059:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.097064:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205915264:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204965.097067:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097068:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097070:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205915264/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097073:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204965.097075:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097076:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204965.097078:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204965.097079:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204965.097080:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204965.097081:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204965.097083:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788204965.097085:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204965.097086:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204965.097087:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204965.097089:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204965.097090:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097091:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097096:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204965.097097:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097098:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204965.097099:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204965.097100:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204965.097102:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097106:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.097107:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097108:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204965.097109:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204965.097110:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204965.097112:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915392:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204965.097113:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204965.097114:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204965.097115:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097116:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097118:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204965.097119:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204965.097121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204965.097123:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204965.097125:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204965.097127:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915392, portal 10 00000100:00000001:3.0:1788204965.097140:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204965.097142:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204965.097147:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097151:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204965.097152:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204965.097154:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204965.097156:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205915392, offset 0 00000400:00000200:3.0:1788204965.097158:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097160:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097163:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204965.097165:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204965.097167:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204965.097168:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204965.097170:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097172:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59007:59007:256:4294967295] 192.168.201.151@tcp LPNI seq info [58987:58987:8:4294967295] 00000400:00000200:3.0:1788204965.097174:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097177:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204965.097180:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097182:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204965.097184:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204965.097186:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204965.097189:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204965.097190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788204965.097193:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097194:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204965.097196:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204965.097198:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204965.097199:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204965.097200:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204965.097202:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204965.097204:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097209:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097210:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097214:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097215:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204965.097217:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204965.097219:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204965.097221:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204965.097222:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301092506 00000100:00000001:3.0:1788204965.097224:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097225:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301092506 00000100:00000001:3.0:1788204965.097227:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097228:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204965.097229:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097230:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204965.097232:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204965.097236:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204965.097237:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204965.097237:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204965.097239:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204965.097240:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204965.097241:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204965.097243:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204965.097247:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097250:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204965.097251:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097252:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097253:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204965.097255:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097258:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204965.097260:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097264:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204965.097266:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097267:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097268:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097269:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204965.097270:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204965.097272:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.097274:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204965.097275:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204965.097277:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.097281:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915392:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204965.097283:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204965.097285:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097288:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.097289:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097290:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204965.097291:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204965.097292:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204965.097294:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915520:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204965.097295:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204965.097296:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204965.097297:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097298:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097300:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204965.097301:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204965.097303:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204965.097304:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204965.097306:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204965.097308:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915520, portal 4 00000100:00000001:3.0:1788204965.097309:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204965.097310:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204965.097313:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097317:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204965.097318:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204965.097319:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204965.097320:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205915520, offset 0 00000400:00000200:3.0:1788204965.097323:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097324:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097326:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204965.097328:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204965.097329:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204965.097330:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204965.097332:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097334:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59008:59008:256:4294967295] 192.168.201.151@tcp LPNI seq info [58988:58988:8:4294967295] 00000400:00000200:3.0:1788204965.097336:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097339:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204965.097342:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097343:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204965.097345:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204965.097347:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204965.097349:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204965.097350:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788204965.097353:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097354:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204965.097355:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204965.097357:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204965.097358:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204965.097359:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204965.097361:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204965.097381:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097385:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097388:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097391:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097392:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204965.097394:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204965.097396:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204965.097398:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204965.097399:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301092393 00000100:00000001:3.0:1788204965.097401:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097402:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301092393 00000100:00000001:3.0:1788204965.097403:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097404:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204965.097405:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097407:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204965.097408:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204965.097412:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204965.097413:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204965.097414:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204965.097421:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204965.097421:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204965.097423:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204965.097424:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204965.097429:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097433:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204965.097433:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097434:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097435:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204965.097438:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097442:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204965.097444:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097447:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204965.097448:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097449:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097450:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097450:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204965.097452:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204965.097453:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.097455:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204965.097456:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204965.097458:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.097461:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915520:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204965.097464:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204965.097465:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097469:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.097470:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097471:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204965.097472:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204965.097473:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204965.097475:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915648:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204965.097476:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204965.097477:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204965.097478:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204965.097483:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097486:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204965.097488:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204965.097490:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204965.097493:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204965.097495:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204965.097498:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915648, portal 4 00000100:00000001:3.0:1788204965.097499:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204965.097501:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204965.097506:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097512:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204965.097514:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204965.097516:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204965.097518:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205915648, offset 0 00000400:00000200:3.0:1788204965.097522:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097525:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097529:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204965.097531:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204965.097533:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204965.097535:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204965.097536:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204965.097538:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59009:59009:256:4294967295] 192.168.201.151@tcp LPNI seq info [58989:58989:8:4294967295] 00000400:00000200:3.0:1788204965.097541:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204965.097545:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204965.097547:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097549:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204965.097552:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204965.097553:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204965.097556:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204965.097557:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788204965.097560:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204965.097561:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204965.097563:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204965.097565:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204965.097567:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204965.097568:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204965.097569:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204965.097571:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097576:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097577:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097581:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097582:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204965.097584:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204965.097586:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204965.097587:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204965.097588:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301092399 00000100:00000001:3.0:1788204965.097590:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097591:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301092399 00000100:00000001:3.0:1788204965.097592:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301096449 00000100:00000001:3.0:1788204965.097593:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204965.097594:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097596:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204965.097597:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204965.097601:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204965.097602:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204965.097603:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204965.097604:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204965.097605:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204965.097606:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204965.097608:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204965.097613:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097616:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204965.097617:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097618:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097619:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204965.097621:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097624:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204965.097626:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097629:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204965.097631:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097632:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204965.097633:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097634:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204965.097635:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204965.097636:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.097638:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204965.097639:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204965.097641:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.097648:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915648:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204965.097651:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097652:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097655:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205915392/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097661:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204965.097663:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097665:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204965.097667:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204965.097669:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204965.097670:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204965.097672:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204965.097673:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788204965.097675:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204965.097676:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204965.097678:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204965.097680:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204965.097681:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097683:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097686:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205915520/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097691:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204965.097692:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097693:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204965.097694:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204965.097695:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204965.097696:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204965.097697:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204965.097698:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788204965.097699:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204965.097700:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204965.097700:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204965.097701:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204965.097702:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097703:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204965.097705:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205915648/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204965.097708:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204965.097709:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204965.097710:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204965.097711:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204965.097712:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204965.097712:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204965.097713:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204965.097714:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788204965.097715:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204965.097716:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204965.097716:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204965.097717:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204965.097718:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097719:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.097721:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204965.097722:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.097723:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204965.097724:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204965.288528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204965.288535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204965.288540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204965.288541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204965.288543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204965.288544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204965.288547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204965.288548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204965.288564:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204965.288566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204965.288568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204965.608425:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204965.608433:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204965.608437:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204965.608451:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204965.608456:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204965.608463:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.608467:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.608470:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204965.608473:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.056470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204966.056480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204966.056483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.056489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204966.056493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204966.056498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.056500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204966.056504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.056507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204966.120502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204966.120509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204966.120511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204966.120517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204966.120520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204966.120524:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204966.120527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204966.120530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204966.120533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204966.120537:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204966.120542:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204966.120543:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204966.120546:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204966.120549:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204966.120551:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204966.120554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204966.120555:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204966.120557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204966.312477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204966.312484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204966.312491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204966.312492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204966.312495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204966.312497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204966.312500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204966.312501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204966.312568:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204966.312572:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204966.312576:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204966.632463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204966.632471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204966.632472:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204966.632476:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204966.632479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204966.632482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204966.632484:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204966.632485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204966.632487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204967.080486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204967.080493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204967.080496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204967.080501:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204967.080505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204967.080509:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204967.080511:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204967.080513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204967.080516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204967.144497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204967.144502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204967.144505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204967.144506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204967.144509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204967.144510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204967.144515:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204967.144516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204967.144520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204967.144521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204967.144523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204967.144525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.144526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204967.144537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.144541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204967.144542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204967.144545:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204967.144548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204967.336498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204967.336504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204967.336508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204967.336509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204967.336511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204967.336512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204967.336514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204967.336515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204967.336527:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204967.336528:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204967.336530:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204967.656449:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204967.656456:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204967.656458:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204967.656463:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204967.656467:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204967.656470:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.656473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204967.656476:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.656479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204968.104465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204968.104472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204968.104474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204968.104479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204968.104483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204968.104486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204968.104488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204968.104491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204968.104493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204968.168468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204968.168469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204968.168477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204968.168477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204968.168479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204968.168480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204968.168485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204968.168486:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204968.168490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204968.168491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204968.168493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204968.168495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204968.168497:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204968.168500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204968.168503:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204968.168507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204968.168510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204968.168513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204968.360447:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204968.360454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204968.360460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204968.360461:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204968.360464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204968.360466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204968.360469:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204968.360470:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204968.360525:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204968.360528:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204968.360531:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204968.680412:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204968.680420:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204968.680422:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204968.680432:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204968.680435:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204968.680439:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204968.680441:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204968.680444:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204968.680446:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204969.128484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204969.128490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204969.128492:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204969.128497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204969.128500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204969.128503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204969.128506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204969.128508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204969.128511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204969.192459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204969.192459:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204969.192465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204969.192465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204969.192468:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204969.192469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204969.192475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204969.192475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204969.192481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204969.192481:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204969.192485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.192485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.192489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204969.192492:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.192494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204969.192496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204969.192498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204969.192500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204969.384508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204969.384513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204969.384516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204969.384517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204969.384519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204969.384519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204969.384521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204969.384522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204969.384535:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204969.384537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204969.384539:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204969.704424:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788204969.704428:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301096449, at 4301096960 00000100:00000040:1.0:1788204969.704428:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301096449, at 4301096960 00000100:00000001:3.0:1788204969.704434:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204969.704438:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204969.704442:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204969.704444:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000001:3.0:1788204969.704445:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788204969.704448:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788204969.704450:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.704454:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204969.704456:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.704458:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204970.152516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204970.152526:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204970.152529:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204970.152536:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204970.152540:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204970.152544:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204970.152547:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204970.152550:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204970.152554:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204970.216477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204970.216477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204970.216484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204970.216485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204970.216488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.216488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204970.216495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204970.216496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204970.216503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204970.216503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788204970.216504:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204970.216508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.216508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.216516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204970.216520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.216526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204970.216527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204970.216527:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204970.216532:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204970.216534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216539:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204970.216542:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204970.216547:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204970.216552:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6425 00000400:00000200:2.0:1788204970.216560:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204970.216565:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204970.216567:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204970.216570:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216572:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204970.216574:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216575:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204970.216578:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204970.216579:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204970.216581:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204970.216583:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204970.216585:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204970.216586:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204970.216588:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216589:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204970.216590:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216591:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204970.216593:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204970.216596:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216599:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204970.216601:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204970.216604:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204970.216606:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216609:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204970.216611:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204970.216619:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788204970.216623:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204970.216626:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204970.216628:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216631:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204970.216633:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204970.216637:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204970.216643:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788204970.216648:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204970.216655:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204970.216662:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204970.216665:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204970.216668:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204970.216688:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204970.216691:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.216693:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204970.216694:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204970.216695:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204970.216697:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204970.216699:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6425 00000400:00000200:2.0:1788204970.216704:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204970.216707:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204970.216709:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204970.216711:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204970.216712:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204970.216717:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204970.216718:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204970.216723:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204970.216723:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204970.216728:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204970.216731:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204970.216735:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000040:3.0:1788204970.216737:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204970.216738:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204970.216742:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204970.216748:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788204970.216755:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204970.216763:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1788204970.216764:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204970.216768:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216769:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204970.216774:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204970.216775:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204970.216777:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788204970.216780:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1788204970.216781:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788204970.216789:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204970.216790:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204970.216799:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204970.216802:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788204970.216804:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205915776:192.168.201.151@tcp:250:kworker.0 02000000:00000001:2.0:1788204970.216807:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204970.216809:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204970.216816:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216818:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204970.216818:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204970.216822:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204970.216824:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204970.216827:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788204970.216829:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216831:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204970.216834:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000010:3.0:1788204970.216836:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000020:00000040:2.0:1788204970.216837:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204970.216845:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204970.216846:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204970.216854:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.216858:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788204970.216862:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788204970.216863:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788204970.216872:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 02000000:00000001:2.0:1788204970.216873:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204970.216882:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 02000000:00000010:2.0:1788204970.216883:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204970.216891:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204970.216895:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915776, portal 25 00000100:00100000:2.0:1788204970.216901:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204970.216902:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204970.216906:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000001:2.0:1788204970.216917:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788204970.216922:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204970.216923:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204970.216933:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.216964:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788204970.216970:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.216972:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788204970.216977:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204970.216978:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00080000:2.0:1788204970.216986:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204970.216989:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1788204970.216989:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205915776, offset 0 00000100:00080000:2.0:1788204970.216996:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204970.217000:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204970.217007:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6425 00000400:00000200:3.0:1788204970.217007:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204970.217028:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204970.217031:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204970.217032:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204970.217037:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788204970.217039:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.217055:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204970.217055:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204970.217064:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788204970.217065:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204970.217070:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204970.217077:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788204970.217080:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204970.217082:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:2.0:1788204970.217087:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788204970.217088:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59010:59010:256:4294967295] 192.168.201.151@tcp LPNI seq info [58990:58990:8:4294967295] 00000020:00000040:2.0:1788204970.217093:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204970.217097:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788204970.217098:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788204970.217101:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204970.217105:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204970.217108:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204970.217118:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204970.217124:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788204970.217125:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204970.217135:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:3.0:1788204970.217137:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204970.217139:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204970.217145:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000010:3.0:1788204970.217145:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000100:00080000:2.0:1788204970.217155:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204970.217158:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204970.217163:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204970.217167:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1788204970.217167:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788204970.217175:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.217178:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204970.217180:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:3.0:1788204970.217180:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000008:00000001:2.0:1788204970.217185:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.217189:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204970.217191:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000800:00000200:3.1:1788204970.217196:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:2.0:1788204970.217197:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.1:1788204970.217207:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 02000000:00000001:2.0:1788204970.217208:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204970.217214:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204970.217220:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:3.0:1788204970.217226:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788204970.217227:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204970.217235:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204970.217236:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 02000000:00000010:2.0:1788204970.217249:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000400:00000200:3.0:1788204970.217252:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204970.217253:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204970.217263:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204970.217268:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204970.217272:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204970.217277:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204970.217277:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788204970.217293:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204970.217294:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000100:00080000:2.0:1788204970.217295:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204970.217300:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00080000:2.0:1788204970.217301:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204970.217304:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204970.217306:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204970.217310:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204970.217310:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788204970.217323:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6425 00000100:00000200:3.0:1788204970.217324:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204970.217336:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204970.217350:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204970.217357:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788204970.217358:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204970.217382:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217389:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217390:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204970.217397:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204970.217401:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204970.217404:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301101570 00000100:00000010:3.0:1788204970.217409:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f41e0. 00000100:00000001:3.0:1788204970.217417:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788204970.217423:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301101570 00000100:00000040:2.0:1788204970.217424:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000010:3.0:1788204970.217432:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4a20. 00000100:00000001:2.0:1788204970.217433:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204970.217446:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1788204970.217447:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.217454:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204970.217456:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788204970.217460:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204970.217463:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788204970.217463:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217471:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788204970.217472:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204970.217478:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00100000:3.0:1788204970.217482:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204970.217485:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000001:3.0:1788204970.217497:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788204970.217498:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000001:3.0:1788204970.217500:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204970.217502:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204970.217506:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000100:00000040:2.0:1788204970.217509:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204970.217509:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204970.217515:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788204970.217520:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788204970.217525:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204970.217527:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204970.217536:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204970.217548:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204970.217559:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000200:3.0:1788204970.217561:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204970.217562:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204970.217569:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204970.217576:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204970.217578:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204970.217585:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204970.217586:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217598:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204970.217599:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204970.217601:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788204970.217605:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1788204970.217607:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788204970.217615:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217618:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204970.217624:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.217630:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204970.217635:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204970.217641:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204970.217646:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204970.217646:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204970.217649:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217655:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204970.217656:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204970.217662:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204970.217664:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788204970.217665:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217669:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788204970.217671:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:3.0:1788204970.217674:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204970.217675:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217677:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204970.217680:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788204970.217682:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204970.217687:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204970.217695:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788204970.217705:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204970.217710:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.217710:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204970.217725:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.217738:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205915776:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204970.217746:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217748:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.217753:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204970.217754:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6435) 00000100:00000040:3.0:1788204970.217764:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205915776/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217774:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204970.217776:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204970.217779:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204970.217782:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204970.217785:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204970.217787:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204970.217788:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204970.217790:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788204970.217794:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204970.217795:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204970.217797:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788204970.217799:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204970.217800:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.217802:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.217809:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204970.217810:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217812:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204970.217813:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204970.217815:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204970.217817:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217822:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204970.217823:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.217824:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204970.217826:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204970.217827:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204970.217829:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915904:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204970.217831:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204970.217833:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204970.217834:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.217835:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204970.217837:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204970.217838:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204970.217841:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204970.217843:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204970.217846:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204970.217848:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205915904, portal 10 00000100:00000001:3.0:1788204970.217849:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204970.217850:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788204970.217855:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217859:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204970.217861:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204970.217862:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204970.217864:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205915904, offset 0 00000400:00000200:3.0:1788204970.217866:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204970.217868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.217871:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204970.217873:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204970.217875:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204970.217877:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204970.217878:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204970.217880:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59011:59011:256:4294967295] 192.168.201.151@tcp LPNI seq info [58991:58991:8:4294967295] 00000400:00000200:3.0:1788204970.217883:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.217887:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204970.217892:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.217895:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204970.217899:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204970.217901:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204970.217906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204970.217909:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1788204970.217913:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.217915:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204970.217918:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204970.217921:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204970.217924:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204970.217926:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204970.217929:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204970.217932:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217938:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.217940:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217944:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217945:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204970.217948:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.217950:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204970.217952:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204970.217954:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301097627 00000100:00000001:3.0:1788204970.217956:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.217957:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301097627 00000100:00000001:3.0:1788204970.217959:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.217960:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204970.217961:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.217962:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204970.217964:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204970.217969:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204970.217969:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204970.217970:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204970.217972:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204970.217973:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204970.217974:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204970.217976:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204970.217981:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217988:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204970.217989:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.217990:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.217991:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204970.217993:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.217996:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204970.217999:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218002:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204970.218003:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218004:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.218005:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218006:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204970.218007:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204970.218009:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.218010:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204970.218011:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204970.218013:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.218018:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205915904:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204970.218020:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204970.218022:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218026:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204970.218027:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.218029:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204970.218030:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204970.218031:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204970.218033:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916032:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204970.218035:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204970.218037:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204970.218037:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.218039:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204970.218052:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204970.218054:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204970.218057:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204970.218059:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204970.218061:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204970.218063:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916032, portal 4 00000100:00000001:3.0:1788204970.218065:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204970.218066:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788204970.218070:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218074:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204970.218075:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204970.218077:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204970.218078:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205916032, offset 0 00000400:00000200:3.0:1788204970.218080:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204970.218082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.218084:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204970.218086:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204970.218088:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204970.218090:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204970.218091:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204970.218093:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59012:59012:256:4294967295] 192.168.201.151@tcp LPNI seq info [58992:58992:8:4294967295] 00000400:00000200:3.0:1788204970.218096:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.218100:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204970.218103:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.218105:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204970.218107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204970.218109:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204970.218112:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204970.218113:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1788204970.218116:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.218117:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204970.218119:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204970.218121:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204970.218123:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204970.218124:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204970.218125:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204970.218127:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218132:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.218134:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218137:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.218138:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204970.218140:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.218143:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204970.218144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204970.218146:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301097513 00000100:00000001:3.0:1788204970.218148:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.218150:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301097513 00000100:00000001:3.0:1788204970.218152:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.218154:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204970.218156:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.218157:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204970.218161:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204970.218170:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204970.218171:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204970.218173:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204970.218175:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204970.218177:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204970.218179:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204970.218182:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204970.218189:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218193:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204970.218194:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218196:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.218197:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204970.218199:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218203:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204970.218205:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218209:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204970.218211:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218212:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.218212:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218213:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204970.218215:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204970.218216:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.218218:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204970.218219:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204970.218222:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.218226:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916032:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204970.218229:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204970.218231:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218234:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204970.218235:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.218237:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204970.218238:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204970.218240:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204970.218241:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916160:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204970.218243:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204970.218244:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204970.218245:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204970.218246:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204970.218248:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204970.218250:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204970.218252:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204970.218253:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000400:00000010:3.0:1788204970.218255:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204970.218257:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916160, portal 4 00000100:00000001:3.0:1788204970.218258:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204970.218259:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204970.218263:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218270:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204970.218272:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204970.218274:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 00000100:00000200:3.0:1788204970.218276:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205916160, offset 0 00000400:00000200:3.0:1788204970.218280:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204970.218284:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.218288:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204970.218291:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204970.218293:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204970.218296:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204970.218298:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204970.218301:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59013:59013:256:4294967295] 192.168.201.151@tcp LPNI seq info [58993:58993:8:4294967295] 00000400:00000200:3.0:1788204970.218305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204970.218310:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204970.218314:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.218316:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000800:00000200:3.0:1788204970.218318:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204970.218320:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204970.218323:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204970.218324:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1788204970.218327:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204970.218328:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:3.0:1788204970.218330:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204970.218332:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204970.218334:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:3.0:1788204970.218335:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:3.0:1788204970.218337:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204970.218339:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218344:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.218346:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218350:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.218351:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204970.218353:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204970.218355:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204970.218357:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204970.218359:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301097519 00000100:00000001:3.0:1788204970.218360:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.218362:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301097519 00000100:00000001:3.0:1788204970.218378:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301101570 00000100:00000001:3.0:1788204970.218379:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204970.218381:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.218382:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204970.218384:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204970.218389:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204970.218390:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204970.218391:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000400:00000200:3.0:1788204970.218393:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204970.218394:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:3.0:1788204970.218395:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204970.218397:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204970.218402:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218406:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204970.218406:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218407:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.218408:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204970.218410:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218413:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204970.218415:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218419:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204970.218420:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218421:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204970.218422:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218423:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204970.218425:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204970.218426:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.218428:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204970.218429:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204970.218431:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.218435:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916160:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204970.218437:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218438:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.218440:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205915904/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218444:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204970.218445:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204970.218446:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204970.218448:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204970.218448:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204970.218449:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204970.218450:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204970.218452:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788204970.218453:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204970.218454:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204970.218455:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204970.218457:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204970.218458:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218459:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.218461:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205916032/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218468:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204970.218469:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204970.218470:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204970.218471:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204970.218472:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204970.218473:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204970.218474:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204970.218474:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:3.0:1788204970.218476:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204970.218476:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204970.218477:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788204970.218478:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204970.218478:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218479:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204970.218481:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205916160/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204970.218484:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204970.218484:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204970.218485:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204970.218486:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204970.218487:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204970.218488:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204970.218489:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204970.218489:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:3.0:1788204970.218490:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204970.218491:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204970.218492:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204970.218493:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204970.218494:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218494:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.218497:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204970.218497:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.218498:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204970.218499:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408440:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204970.408451:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204970.408464:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204970.408466:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204970.408471:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408474:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204970.408477:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408479:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408482:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204970.408488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204970.408492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204970.408493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204970.408564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204970.408570:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204970.408575:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204970.728433:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204970.728440:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204970.728442:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204970.728448:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204970.728452:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204970.728455:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.728458:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204970.728461:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.728463:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.176446:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204971.176451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204971.176453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204971.176457:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204971.176459:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.176462:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.176464:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.176465:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.176467:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204971.240490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204971.240491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204971.240500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204971.240501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204971.240503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204971.240507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204971.240510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204971.240514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204971.240515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204971.240523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204971.240523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204971.240531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204971.240531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204971.240540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204971.240544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204971.240547:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204971.240550:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204971.240553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204971.432511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204971.432517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204971.432520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204971.432521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204971.432523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204971.432524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204971.432526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204971.432527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204971.432540:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204971.432541:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204971.432543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204971.752473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204971.752481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204971.752483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204971.752489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788204971.752493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204971.752496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204971.752499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204971.752501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204971.752503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204972.200490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204972.200498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204972.200500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204972.200506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204972.200510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204972.200514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204972.200518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204972.200520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204972.200523:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204972.264457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204972.264459:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204972.264464:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204972.264465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204972.264466:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204972.264468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204972.264472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204972.264473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204972.264477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204972.264478:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204972.264481:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.264481:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.264487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204972.264489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.264490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204972.264492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204972.264494:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204972.264496:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204972.456542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204972.456548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204972.456551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204972.456552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204972.456560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204972.456561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204972.456563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204972.456564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204972.456616:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204972.456619:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204972.456621:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204972.776470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204972.776476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204972.776478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204972.776483:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204972.776486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204972.776494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.776496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204972.776499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.776503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.224442:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204973.224448:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204973.224449:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204973.224453:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204973.224455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204973.224458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204973.224460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204973.224462:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204973.224464:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204973.288462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204973.288462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204973.288468:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204973.288470:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204973.288474:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204973.288476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204973.288477:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204973.288479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204973.288481:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204973.288483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204973.288484:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204973.288486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204973.288488:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204973.288490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204973.288491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204973.288493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204973.288495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204973.288497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204973.480515:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204973.480518:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204973.480521:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480526:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204973.480528:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480530:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480532:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204973.480537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204973.480541:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204973.480543:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204973.480544:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480546:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204973.480547:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480549:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480551:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204973.480555:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204973.480557:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204973.480558:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480560:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204973.480561:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480562:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204973.480564:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204973.480582:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204973.480584:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204973.480586:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204973.800471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204973.800477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204973.800478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204973.800482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204973.800484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204973.800487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204973.800489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204973.800491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204973.800493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204974.248458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204974.248463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204974.248464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204974.248468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204974.248471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204974.248474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.248475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204974.248477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.248479:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204974.312479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204974.312486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204974.312491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204974.312492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204974.312502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204974.312502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204974.312510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204974.312511:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204974.312516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204974.312521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204974.312522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204974.312526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204974.312529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.312539:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204974.312544:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204974.312548:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204974.312550:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204974.312555:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204974.504471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204974.504478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204974.504484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204974.504485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204974.504488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204974.504489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204974.504492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204974.504493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204974.504555:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204974.504559:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204974.504562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204974.824458:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204974.824460:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301101570, at 4301102080 00000100:00000040:0.0:1788204974.824463:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301101570, at 4301102080 00000100:00000001:3.0:1788204974.824465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204974.824467:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204974.824472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204974.824474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1788204974.824475:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000001:3.0:1788204974.824477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204974.824478:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000001:3.0:1788204974.824482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204974.824485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204974.824487:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204975.272425:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204975.272431:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204975.272433:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204975.272436:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204975.272438:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204975.272441:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204975.272443:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204975.272444:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204975.272446:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.336466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204975.336472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204975.336473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204975.336477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204975.336480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204975.336484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.336486:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.336488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.336490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204975.336504:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204975.336511:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204975.336511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204975.336514:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788204975.336515:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204975.336518:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204975.336520:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788204975.336521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788204975.336525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204975.336528:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788204975.336529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204975.336533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204975.336535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204975.336537:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6430 00000100:00000001:2.0:1788204975.336539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204975.336549:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204975.336555:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204975.336558:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204975.336562:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336566:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.336568:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336571:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204975.336574:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204975.336576:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204975.336579:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204975.336582:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204975.336585:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204975.336587:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204975.336590:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336592:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.336594:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336596:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204975.336599:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204975.336603:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336605:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204975.336608:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204975.336611:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204975.336614:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.336616:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204975.336619:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204975.336628:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788204975.336631:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204975.336635:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204975.336637:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.336639:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204975.336642:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204975.336645:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204975.336653:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788204975.336659:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204975.336667:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204975.336676:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204975.336681:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204975.336686:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204975.336708:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204975.336711:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204975.336715:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204975.336716:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204975.336718:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204975.336721:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204975.336724:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6430 00000400:00000200:0.0:1788204975.336729:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204975.336732:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1788204975.336733:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204975.336735:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204975.336737:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204975.336738:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204975.336746:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204975.336747:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204975.336752:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.336757:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204975.336759:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204975.336761:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204975.336761:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.336765:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788204975.336766:0:30130:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788204975.336768:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788204975.336769:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 02000000:00000001:3.0:1788204975.336770:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204975.336773:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1788204975.336776:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205916288:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788204975.336777:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204975.336780:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1788204975.336782:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204975.336784:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1788204975.336787:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788204975.336788:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.336792:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1788204975.336795:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336798:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:3.0:1788204975.336799:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 00000100:00000040:0.0:1788204975.336801:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204975.336803:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.336805:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204975.336809:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000400:00000200:3.0:1788204975.336813:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204975.336815:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:3.0:1788204975.336819:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f52c0. 00000100:00000001:0.0:1788204975.336820:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204975.336825:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1788204975.336829:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788204975.336836:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916288, portal 25 02000000:00000001:0.0:1788204975.336839:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.336840:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204975.336843:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 02000000:00000001:0.0:1788204975.336844:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.336848:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:3.0:1788204975.336852:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788204975.336853:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204975.336859:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.336864:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.336869:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:0.0:1788204975.336870:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000040:3.0:1788204975.336873:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788204975.336875:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:3.0:1788204975.336879:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f45d8. 02000000:00000001:0.0:1788204975.336880:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204975.336883:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204975.336886:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205916288, offset 0 00000100:00000001:0.0:1788204975.336887:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204975.336889:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204975.336893:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.336896:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000010:0.0:1788204975.336899:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788204975.336905:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.336906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:0.0:1788204975.336914:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204975.336915:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204975.336922:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204975.336929:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204975.336932:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788204975.336936:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204975.336937:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.336940:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.336945:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59014:59014:256:4294967295] 192.168.201.151@tcp LPNI seq info [58994:58994:8:4294967295] 00000100:00000040:0.0:1788204975.336950:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204975.336976:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204975.336979:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204975.336980:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204975.336986:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204975.336988:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204975.336990:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204975.336996:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204975.336999:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:0.0:1788204975.337002:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6430 00000400:00000200:0.0:1788204975.337011:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:3.0:1788204975.337011:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204975.337018:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecf00. 00000400:00000200:0.0:1788204975.337019:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204975.337023:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1788204975.337026:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788204975.337030:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.1:1788204975.337033:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788204975.337035:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.337040:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.337042:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204975.337045:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204975.337046:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000001:3.0:1788204975.337048:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204975.337049:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204975.337053:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204975.337055:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788204975.337059:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:3.0:1788204975.337059:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301106689 00000020:00000001:0.0:1788204975.337065:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000010:3.0:1788204975.337066:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f48a0. 00000100:00000040:0.0:1788204975.337074:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204975.337074:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788204975.337079:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204975.337083:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301106689 00000100:00000040:0.0:1788204975.337084:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.337087:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000010:3.0:1788204975.337088:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4000. 00000020:00000040:0.0:1788204975.337092:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000001:3.0:1788204975.337093:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788204975.337098:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204975.337098:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.337101:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.337104:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.337105:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.337107:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788204975.337109:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204975.337113:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.337114:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204975.337116:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1788204975.337119:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.337119:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204975.337121:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204975.337124:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204975.337124:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1788204975.337129:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000040:3.0:1788204975.337129:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788204975.337136:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.337141:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.337141:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.337144:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.337146:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204975.337147:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1788204975.337148:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 02000000:00000001:3.0:1788204975.337149:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204975.337155:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00100000:3.0:1788204975.337155:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916416:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204975.337161:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204975.337162:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204975.337163:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204975.337166:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204975.337167:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.337167:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788204975.337171:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:3.0:1788204975.337171:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:0.0:1788204975.337175:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204975.337175:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204975.337180:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.337180:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204975.337185:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000010:0.0:1788204975.337188:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000400:00000010:3.0:1788204975.337191:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788204975.337193:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204975.337196:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916416, portal 10 00000100:00000001:3.0:1788204975.337200:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788204975.337203:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204975.337203:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788204975.337213:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204975.337229:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204975.337232:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.337238:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204975.337240:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204975.337245:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204975.337250:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f44c8. 00000100:00000200:3.0:1788204975.337254:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205916416, offset 0 00000100:00000001:0.0:1788204975.337257:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.337262:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204975.337263:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204975.337269:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204975.337272:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204975.337273:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204975.337276:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204975.337277:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204975.337280:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6430 00000400:00000200:3.0:1788204975.337285:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788204975.337287:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204975.337290:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204975.337294:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204975.337296:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204975.337298:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204975.337301:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59015:59015:255:4294967295] 192.168.201.151@tcp LPNI seq info [58995:58995:7:4294967295] 00000400:00000200:3.0:1788204975.337305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204975.338054:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204975.338058:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204975.338062:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.338065:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.338067:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.338069:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204975.338072:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204975.338073:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788204975.338075:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204975.338077:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788204975.338079:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788204975.338082:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204975.338084:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.338087:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204975.338089:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204975.338091:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204975.338094:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788204975.338097:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204975.338099:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204975.338102:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204975.338105:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204975.338108:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.338110:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204975.338112:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788204975.338115:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.338117:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204975.338121:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788204975.338124:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788204975.338127:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204975.338128:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204975.338130:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204975.338131:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204975.338133:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204975.338137:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788204975.338141:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204975.338147:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204975.338156:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204975.338158:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204975.338162:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204975.338170:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204975.338175:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6440) 00000400:00000200:3.0:1788204975.340275:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204975.340286:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204975.340295:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ece00. 00000800:00000200:3.0:1788204975.340301:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204975.340306:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204975.340312:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204975.340317:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204975.340322:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204975.340325:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301102749 00000100:00000001:3.0:1788204975.340328:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340330:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301102749 00000100:00000001:3.0:1788204975.340333:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340335:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.340337:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340341:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340343:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.340348:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204975.340350:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204975.340351:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204975.340353:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204975.340354:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204975.340359:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204975.340380:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.340382:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.340390:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204975.340393:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204975.340394:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204975.340396:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916544:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204975.340399:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204975.340402:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204975.340404:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.340405:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204975.340409:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204975.340410:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.340413:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204975.340416:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5a50. 00000400:00000010:3.0:1788204975.340422:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4088. 00000100:00000200:3.0:1788204975.340427:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916544, portal 4 00000100:00000001:3.0:1788204975.340430:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204975.340431:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204975.340439:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204975.340445:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204975.340447:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204975.340450:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4198. 00000100:00000200:3.0:1788204975.340453:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205916544, offset 0 00000400:00000200:3.0:1788204975.340458:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204975.340465:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204975.340471:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204975.340475:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204975.340479:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204975.340482:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204975.340485:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204975.340487:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59016:59016:254:4294967295] 192.168.201.151@tcp LPNI seq info [58996:58996:6:4294967295] 00000400:00000200:3.0:1788204975.340492:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204975.340505:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204975.340511:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204975.340514:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecb00. 00000800:00000200:3.0:1788204975.340516:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204975.340518:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204975.340520:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204975.340522:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204975.340524:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204975.340526:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301102636 00000100:00000001:3.0:1788204975.340529:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340531:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301102636 00000100:00000001:3.0:1788204975.340533:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340535:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.340537:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340539:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204975.340543:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204975.340550:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.340551:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.340553:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204975.340555:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204975.340557:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204975.340559:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916672:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204975.340562:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204975.340564:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204975.340565:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204975.340567:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204975.340571:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204975.340572:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204975.340576:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204975.340578:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5f20. 00000400:00000010:3.0:1788204975.340581:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204975.340584:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916672, portal 4 00000100:00000001:3.0:1788204975.340585:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204975.340586:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204975.340592:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204975.340599:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204975.340601:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204975.340604:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4dd0. 00000100:00000200:3.0:1788204975.340606:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205916672, offset 0 00000400:00000200:3.0:1788204975.340609:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204975.340612:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204975.340614:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204975.340616:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204975.340618:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204975.340619:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204975.340621:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204975.340623:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59017:59017:253:4294967295] 192.168.201.151@tcp LPNI seq info [58997:58997:5:4294967295] 00000400:00000200:3.0:1788204975.340625:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204975.340629:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204975.340633:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204975.340636:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204975.340640:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204975.340644:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788204975.340646:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204975.340648:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204975.340651:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204975.340653:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301102642 00000100:00000001:3.0:1788204975.340655:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340657:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301102642 00000100:00000001:3.0:1788204975.340659:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301106689 00000100:00000001:3.0:1788204975.340661:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204975.340663:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340665:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340667:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.340671:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204975.340672:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204975.340674:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204975.340675:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204975.340677:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.340678:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204975.528527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204975.528532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204975.528538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204975.528539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204975.528542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204975.528543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204975.528546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204975.528547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204975.528563:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204975.528566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204975.528568:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204975.848463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204975.848471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204975.848473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204975.848477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204975.848481:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204975.848484:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.848487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.848489:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.848492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.296468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204976.296474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204976.296475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.296480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204976.296482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204976.296485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.296487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.296489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.296491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204976.360447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204976.360447:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204976.360454:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204976.360457:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204976.360461:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204976.360462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204976.360466:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204976.360469:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204976.360470:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204976.360474:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788204976.360475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204976.360476:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204976.360481:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204976.360481:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204976.360486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204976.360490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204976.360492:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204976.360494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204976.552482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204976.552488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204976.552491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204976.552492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204976.552494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204976.552496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204976.552498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204976.552504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204976.552561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204976.552565:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204976.552569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204976.872411:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204976.872417:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204976.872418:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204976.872422:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204976.872424:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204976.872427:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204976.872429:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204976.872430:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204976.872432:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204977.320456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204977.320461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204977.320463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204977.320466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204977.320469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204977.320471:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204977.320473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204977.320474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204977.320477:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204977.384465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204977.384471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204977.384472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204977.384476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204977.384479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204977.384482:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204977.384483:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204977.384485:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204977.384487:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204977.384494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204977.384499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204977.384501:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204977.384504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204977.384507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204977.384510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204977.384512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204977.384515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204977.384517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204977.576484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204977.576490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204977.576494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204977.576496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204977.576498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204977.576499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204977.576502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204977.576504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204977.576519:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204977.576521:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204977.576523:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204977.896502:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204977.896510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204977.896513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204977.896519:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204977.896523:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204977.896527:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204977.896529:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204977.896532:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204977.896534:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204978.344523:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204978.344539:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204978.344542:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204978.344549:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204978.344553:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204978.344557:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204978.344560:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204978.344562:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204978.344566:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204978.408494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204978.408494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204978.408502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204978.408503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204978.408505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.408507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204978.408512:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204978.408513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204978.408517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204978.408519:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204978.408521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.408523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204978.408525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204978.408528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204978.408532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.408532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204978.408536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.408539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1788204978.408696:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:1.0:1788204978.408744:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1788204978.408756:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204978.408760:0:2335:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:1.0:1788204978.408766:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788204978.408771:0:2335:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1788204978.408776:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204978.408781:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecf00. 00000400:00000200:1.0:1788204978.408786:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204978.408791:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788204978.408799:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f45d8 00000400:00000010:1.0:1788204978.408801:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f45d8. 00000100:00000001:1.0:1788204978.408807:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204978.408818:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204978.408845:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204978.408849:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204978.408857:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204978.408859:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788204978.408871:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204978.408874:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ece00. 00000400:00000200:1.0:1788204978.408877:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204978.408880:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788204978.408883:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f44c8 00000400:00000010:1.0:1788204978.408885:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f44c8. 00000100:00000001:3.0:1788204978.408888:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204978.408889:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788204978.408894:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1788204978.408895:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.408909:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204978.408919:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204975/real 1788204978] req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204978.408922:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204978.408929:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204978.408937:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204978.408942:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204978.408947:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f52c0. 00000100:00000001:1.0:1788204978.408948:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204978.408952:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1788204978.408955:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:3.0:1788204978.408958:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000800:00000100:1.0:1788204978.408963:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000100:00000001:3.0:1788204978.408965:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000010:1.0:1788204978.408967:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecb00. 00000400:00000200:1.0:1788204978.408972:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000200:3.0:1788204978.408973:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204978.408980:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788204978.408986:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4198 00000400:00000010:1.0:1788204978.408991:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4198. 00000100:00000001:1.0:1788204978.408996:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204978.408996:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204978.409004:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409012:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204978.409015:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204978.409018:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204978.409020:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1788204978.409022:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409025:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204978.409036:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204978.409037:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204978.409040:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788204978.409048:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788204978.409051:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:1.0:1788204978.409054:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788204978.409057:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000100:00000001:3.0:1788204978.409062:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788204978.409063:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4dd0 00000400:00000010:1.0:1788204978.409067:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4dd0. 00000100:00000001:1.0:1788204978.409071:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788204978.409073:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204978.409079:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409091:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204978.409097:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204978.409099:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204978.409101:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409103:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204978.409106:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205000 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409108:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204978.409113:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204978.409120:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204978.409121:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788204978.409126:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1788204978.409131:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204978.409134:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204978.409139:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204978.409148:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205916288:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204978.409152:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.409154:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204978.409157:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205916288/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409163:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204978.409167:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204978.409172:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788204978.409177:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204978.409181:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204978.409183:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204978.409192:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204978.409195:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788204978.409200:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204978.409203:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204978.409205:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204978.409209:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204978.409210:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409214:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.409216:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204978.409217:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204978.409218:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204978.409223:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204975/real 1788204978] req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204978.409232:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204978.409233:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204978.409236:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5a50. 00000400:00000200:3.0:1788204978.409238:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4088 00000400:00000010:3.0:1788204978.409240:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4088. 00000100:00000001:3.0:1788204978.409243:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204978.409246:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204978.409254:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409261:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204978.409262:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409264:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409265:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204978.409269:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409275:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204978.409279:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409286:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204978.409288:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409290:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409292:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409293:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204978.409297:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204978.409299:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204978.409303:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204978.409306:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204978.409310:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204978.409406:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916544:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204978.409412:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204978.409416:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204975/real 1788204978] req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204978.409424:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204978.409426:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204978.409428:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5f20. 00000400:00000200:3.0:1788204978.409431:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204978.409433:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204978.409436:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204978.409439:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204978.409450:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409458:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204978.409460:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409462:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409463:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204978.409468:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409479:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204978.409484:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409490:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204978.409492:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409493:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409494:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409496:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204978.409499:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204978.409501:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204978.409505:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204978.409507:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204978.409511:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204978.409518:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916672:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204978.409522:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204978.409525:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204975/real 1788204978] req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204978.409533:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204978.409534:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204978.409536:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204978.409538:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204978.409540:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204978.409542:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204978.409545:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204978.409557:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409563:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204978.409565:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409566:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409567:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204978.409570:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409576:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204978.409579:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409585:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204978.409587:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409588:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204978.409589:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409591:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204978.409593:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204978.409595:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204978.409598:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204978.409601:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204978.409604:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204978.409611:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916416:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204978.409615:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409617:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204978.409620:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205916544/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409631:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204978.409633:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204978.409635:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204978.409638:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204978.409640:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204978.409642:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204978.409644:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204978.409645:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788204978.409648:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204978.409649:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204978.409651:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204978.409654:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204978.409655:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409657:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204978.409666:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205916672/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409673:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204978.409674:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204978.409675:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204978.409679:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204978.409680:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204978.409681:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204978.409684:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204978.409685:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788204978.409687:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204978.409688:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204978.409689:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204978.409692:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204978.409693:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409694:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204978.409697:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205916416/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205000 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204978.409702:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204978.409703:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204978.409705:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204978.409707:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204978.409708:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204978.409710:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204978.409712:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204978.409713:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788204978.409717:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204978.409718:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204978.409719:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204978.409721:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204978.409722:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409724:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.409734:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204978.409736:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.409738:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204978.409739:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204978.600501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204978.600510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204978.600518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204978.600519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204978.600523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204978.600524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204978.600528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204978.600529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204978.600590:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204978.600595:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204978.600599:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204978.920499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204978.920508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204978.920511:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204978.920517:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204978.920521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204978.920526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.920529:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204978.920532:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.920535:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204979.368447:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204979.368455:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204979.368457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204979.368462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204979.368465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204979.368468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204979.368471:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204979.368473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204979.368475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204979.432467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204979.432467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204979.432478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204979.432481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204979.432485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788204979.432488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204979.432489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204979.432494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204979.432496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204979.432499:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204979.432500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204979.432503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204979.432505:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204979.432506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204979.432511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204979.432512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204979.432515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204979.432518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204979.624483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204979.624490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204979.624496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204979.624498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204979.624506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204979.624507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204979.624510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204979.624511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204979.624529:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204979.624531:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204979.624533:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204979.944494:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301106689, at 4301107200 00000100:00000040:1.0:1788204979.944495:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301106689, at 4301107200 00000100:00000001:3.0:1788204979.944508:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:1.0:1788204979.944516:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000001:3.0:1788204979.944516:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788204979.944517:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000001:3.0:1788204979.944522:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204979.944530:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204979.944535:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204979.944540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204979.944544:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204979.944547:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204979.944551:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204980.392501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204980.392512:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204980.392514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204980.392520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204980.392523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204980.392527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204980.392530:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204980.392532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204980.392535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.456464:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204980.456469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204980.456474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204980.456476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204980.456478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204980.456479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204980.456487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204980.456487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204980.456492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204980.456498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204980.456500:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204980.456501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204980.456503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204980.456509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204980.456512:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204980.456513:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204980.456517:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204980.456519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204980.456520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204980.456520:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204980.456530:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204980.456531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204980.456537:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204980.456539:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6435 00000400:00000200:2.0:1788204980.456553:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204980.456559:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204980.456562:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204980.456565:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456568:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.456570:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456571:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204980.456575:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204980.456576:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204980.456579:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204980.456582:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204980.456583:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204980.456584:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204980.456586:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.456589:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456590:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204980.456592:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204980.456595:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456596:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204980.456598:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204980.456602:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204980.456604:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.456605:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204980.456607:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204980.456613:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788204980.456616:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204980.456619:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204980.456620:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.456624:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204980.456626:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204980.456630:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204980.456638:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788204980.456645:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204980.456658:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204980.456669:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204980.456675:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204980.456679:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204980.456707:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204980.456710:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204980.456714:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204980.456716:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204980.456718:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204980.456722:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204980.456725:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6435 00000400:00000200:2.0:1788204980.456730:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204980.456735:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204980.456735:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788204980.456740:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204980.456744:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788204980.456745:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456751:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204980.456752:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204980.456754:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456759:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204980.456761:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204980.456764:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204980.456765:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204980.456769:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204980.456772:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204980.456772:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204980.456780:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204980.456790:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456795:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.456799:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456808:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204980.456815:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204980.456820:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788204980.456822:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204980.456827:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.456829:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204980.456830:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204980.456838:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204980.456843:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204980.456844:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204980.456854:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.456857:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.456860:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204980.456881:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204980.456883:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205916800:192.168.201.151@tcp:250:kworker.0 00000008:00000001:2.0:1788204980.456892:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.456896:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204980.456897:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204980.456905:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 02000000:00000001:3.0:1788204980.456907:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1788204980.456910:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204980.456914:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204980.456916:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204980.456922:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.456925:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204980.456927:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204980.456931:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204980.456935:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204980.456942:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 02000000:00000010:3.0:1788204980.456942:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 00000100:00000001:2.0:1788204980.456952:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.456953:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204980.456970:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204980.456972:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204980.456988:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1788204980.456988:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:2.0:1788204980.456994:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204980.457002:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204980.457004:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:2.0:1788204980.457021:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204980.457022:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916800, portal 25 00000100:00080000:2.0:1788204980.457025:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204980.457032:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788204980.457034:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204980.457037:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000001:2.0:1788204980.457038:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204980.457043:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204980.457049:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788204980.457052:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204980.457055:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6435 00000400:00000200:2.0:1788204980.457070:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204980.457078:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204980.457084:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204980.457091:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457093:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.457096:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204980.457096:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788204980.457104:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204980.457110:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204980.457113:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000040:3.0:1788204980.457113:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788204980.457118:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:3.0:1788204980.457120:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000020:00000040:2.0:1788204980.457121:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204980.457128:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:3.0:1788204980.457130:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205916800, offset 0 00000020:00000001:2.0:1788204980.457136:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204980.457147:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204980.457151:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457154:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.457160:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457162:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204980.457172:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788204980.457175:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788204980.457182:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457184:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204980.457186:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204980.457191:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204980.457193:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000001:2.0:1788204980.457201:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.457208:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204980.457210:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204980.457215:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204980.457219:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000008:00000001:2.0:1788204980.457222:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.457226:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.457229:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000010:2.0:1788204980.457236:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000400:00000200:3.0:1788204980.457237:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:2.0:1788204980.457244:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788204980.457249:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59018:59018:256:4294967295] 192.168.201.151@tcp LPNI seq info [58998:58998:8:4294967295] 02000000:00000001:2.0:1788204980.457251:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204980.457254:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.457256:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204980.457258:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204980.457260:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204980.457264:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788204980.457267:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204980.457274:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204980.457281:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204980.457283:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204980.457287:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204980.457294:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204980.457297:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204980.457300:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204980.457302:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204980.457303:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204980.457305:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204980.457308:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6435 00000400:00000200:3.0:1788204980.457347:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204980.457356:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.457400:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204980.457409:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.457416:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204980.457431:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204980.457436:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000400:00000200:2.0:1788204980.457441:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204980.457447:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204980.457450:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204980.457456:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204980.457460:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.457468:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000040:2.0:1788204980.457471:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204980.457475:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204980.457477:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.457483:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457490:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204980.457491:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1788204980.457498:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204980.457501:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000400:00000200:3.0:1788204980.457503:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000020:00000001:2.0:1788204980.457505:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:3.0:1788204980.457509:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000020:00000040:2.0:1788204980.457510:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204980.457517:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204980.457521:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204980.457525:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1788204980.457526:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457532:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204980.457537:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204980.457538:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204980.457543:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204980.457552:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788204980.457561:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204980.457569:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204980.457578:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204980.457586:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204980.457590:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788204980.457592:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.457600:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:3.0:1788204980.457600:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788204980.457606:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204980.457617:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457617:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204980.457625:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457627:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1788204980.457634:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204980.457638:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204980.457640:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204980.457646:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204980.457650:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.457653:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204980.457655:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204980.457659:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204980.457663:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457663:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301111810 02000000:00000010:2.0:1788204980.457675:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000010:3.0:1788204980.457677:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4de0. 00000100:00000001:2.0:1788204980.457681:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457691:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1788204980.457692:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204980.457706:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301111810 00000100:00000010:3.0:1788204980.457717:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 00000100:00000001:2.0:1788204980.457725:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204980.457729:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788204980.457730:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457740:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204980.457741:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457755:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457774:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204980.457782:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204980.457790:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204980/real 1788204980] req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204980.457796:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6445) 00000400:00000001:3.0:1788204980.457811:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204980.457814:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204980.457818:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204980.457822:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204980.457824:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204980.457831:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204980.457835:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204980.457843:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457848:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204980.457850:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457852:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.457854:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204980.457857:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457862:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204980.457882:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457889:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204980.457892:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457894:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.457895:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457898:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204980.457900:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204980.457904:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204980.457908:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204980.457911:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204980.457915:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204980.457922:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205916800:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204980.457927:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457929:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.457932:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205916800/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457938:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204980.457940:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204980.457943:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204980.457946:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204980.457949:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204980.457952:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204980.457954:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204980.457956:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788204980.457960:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204980.457962:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204980.457964:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788204980.457968:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204980.457968:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457971:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.457979:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204980.457980:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.457983:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204980.457984:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204980.457986:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204980.457989:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.457996:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.457998:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458000:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204980.458002:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204980.458005:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204980.458007:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916928:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204980.458011:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204980.458013:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204980.458015:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458016:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458020:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204980.458022:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.458026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204980.458029:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204980.458031:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204980.458035:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205916928, portal 10 00000100:00000001:3.0:1788204980.458037:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204980.458038:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788204980.458044:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458051:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204980.458053:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204980.458055:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204980.458057:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205916928, offset 0 00000400:00000200:3.0:1788204980.458061:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458066:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458070:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204980.458073:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204980.458077:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204980.458079:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204980.458081:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458084:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59019:59019:256:4294967295] 192.168.201.151@tcp LPNI seq info [58999:58999:8:4294967295] 00000400:00000200:3.0:1788204980.458088:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458095:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204980.458099:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458103:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204980.458107:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204980.458110:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204980.458114:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204980.458116:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204980.458120:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458122:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204980.458124:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204980.458128:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204980.458131:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204980.458132:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204980.458134:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204980.458138:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458145:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458148:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458159:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458161:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204980.458164:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204980.458167:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204980.458169:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204980.458170:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301107867 00000100:00000001:3.0:1788204980.458172:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458173:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301107867 00000100:00000001:3.0:1788204980.458175:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458176:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204980.458177:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458178:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204980.458180:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204980/real 1788204980] req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204980.458185:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204980.458186:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204980.458187:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204980.458188:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204980.458189:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204980.458191:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204980.458192:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204980.458197:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458201:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204980.458202:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458203:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458204:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204980.458205:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458210:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204980.458212:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458216:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204980.458217:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458218:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458218:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458219:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204980.458221:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204980.458223:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204980.458224:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204980.458225:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204980.458228:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204980.458232:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205916928:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204980.458234:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204980.458236:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458240:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.458241:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458243:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204980.458244:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204980.458245:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204980.458246:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917056:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204980.458248:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204980.458250:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204980.458251:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458253:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458255:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204980.458256:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.458258:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204980.458260:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204980.458262:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204980.458264:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917056, portal 4 00000100:00000001:3.0:1788204980.458265:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204980.458266:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788204980.458269:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458273:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204980.458274:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204980.458275:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204980.458276:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205917056, offset 0 00000400:00000200:3.0:1788204980.458278:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458280:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458283:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204980.458284:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204980.458286:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204980.458287:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204980.458289:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458291:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59020:59020:256:4294967295] 192.168.201.151@tcp LPNI seq info [59000:59000:8:4294967295] 00000400:00000200:3.0:1788204980.458293:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458297:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204980.458300:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458301:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204980.458304:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204980.458305:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204980.458308:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204980.458322:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204980.458324:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458326:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204980.458328:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204980.458330:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204980.458332:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204980.458333:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204980.458335:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204980.458337:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458342:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458343:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458347:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458348:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204980.458350:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204980.458352:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204980.458354:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204980.458355:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301107753 00000100:00000001:3.0:1788204980.458357:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458358:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301107753 00000100:00000001:3.0:1788204980.458359:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458360:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204980.458361:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458388:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204980.458391:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204980/real 1788204980] req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204980.458395:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204980.458396:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204980.458397:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204980.458399:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204980.458400:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204980.458401:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204980.458403:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204980.458408:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458411:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204980.458412:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458413:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458414:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204980.458415:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458419:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204980.458421:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458425:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204980.458427:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458428:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458428:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458429:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204980.458431:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204980.458432:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204980.458434:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204980.458435:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204980.458437:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204980.458441:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917056:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204980.458443:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204980.458445:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458450:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.458451:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458452:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204980.458454:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204980.458455:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204980.458456:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917184:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204980.458458:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204980.458459:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204980.458460:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204980.458462:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458465:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204980.458467:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204980.458469:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204980.458471:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204980.458473:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204980.458476:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917184, portal 4 00000100:00000001:3.0:1788204980.458477:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204980.458479:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204980.458483:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458489:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204980.458491:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204980.458493:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204980.458495:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205917184, offset 0 00000400:00000200:3.0:1788204980.458499:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458502:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458505:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204980.458508:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204980.458511:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204980.458513:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204980.458515:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204980.458517:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59021:59021:256:4294967295] 192.168.201.151@tcp LPNI seq info [59001:59001:8:4294967295] 00000400:00000200:3.0:1788204980.458520:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204980.458524:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204980.458527:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458529:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204980.458531:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204980.458533:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204980.458535:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204980.458537:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788204980.458539:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204980.458541:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204980.458542:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204980.458544:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204980.458546:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204980.458547:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204980.458549:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204980.458551:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458556:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458557:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458561:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458562:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204980.458564:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204980.458566:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204980.458567:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204980.458569:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301107760 00000100:00000001:3.0:1788204980.458570:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458571:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301107760 00000100:00000001:3.0:1788204980.458573:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301111810 00000100:00000001:3.0:1788204980.458574:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204980.458576:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458577:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204980.458579:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204980/real 1788204980] req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204980.458583:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204980.458584:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204980.458585:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204980.458586:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204980.458587:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204980.458588:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204980.458590:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204980.458595:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458598:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204980.458599:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458600:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458600:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204980.458603:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458609:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204980.458612:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458617:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204980.458619:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458621:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204980.458622:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458623:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204980.458625:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204980.458627:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204980.458631:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204980.458633:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204980.458637:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204980.458644:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917184:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204980.458648:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458650:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458653:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205916928/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458658:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204980.458659:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458661:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204980.458662:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204980.458664:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204980.458665:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204980.458666:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204980.458667:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788204980.458669:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204980.458670:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204980.458671:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788204980.458673:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204980.458674:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458675:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458677:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205917056/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458680:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204980.458681:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458682:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204980.458683:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204980.458684:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204980.458685:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204980.458686:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204980.458687:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:3.0:1788204980.458688:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204980.458689:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204980.458690:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788204980.458691:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204980.458692:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458693:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204980.458695:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205917184/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204980.458698:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204980.458698:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204980.458699:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204980.458700:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204980.458701:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204980.458701:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204980.458702:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204980.458703:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:3.0:1788204980.458704:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204980.458704:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204980.458705:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204980.458706:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204980.458707:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458708:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.458710:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204980.458711:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.458712:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204980.458713:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648460:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204980.648467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204980.648478:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204980.648481:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204980.648484:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648487:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204980.648490:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648492:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648494:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204980.648499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204980.648501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204980.648502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204980.648554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204980.648560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204980.648564:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204980.968468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204980.968476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204980.968478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204980.968481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204980.968484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204980.968486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.968488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.968490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.968492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204981.416481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204981.416488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204981.416490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204981.416495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204981.416499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204981.416502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204981.416505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204981.416507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204981.416510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204981.480469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204981.480469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204981.480475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204981.480476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204981.480477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204981.480479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204981.480482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204981.480485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204981.480486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204981.480490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204981.480491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.480495:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204981.480496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.480497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204981.480502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204981.480502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204981.480506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204981.480510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204981.672536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204981.672543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204981.672549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204981.672550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204981.672553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204981.672554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204981.672557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204981.672558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204981.672575:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204981.672578:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204981.672580:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204981.992472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204981.992487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204981.992489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204981.992495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204981.992498:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204981.992501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.992502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204981.992504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.992506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.440463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204982.440469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204982.440471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204982.440475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204982.440477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204982.440480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204982.440482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204982.440484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204982.440486:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204982.504475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204982.504475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204982.504482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204982.504483:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204982.504487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.504487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.504494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204982.504495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204982.504499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204982.504500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204982.504504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.504504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204982.504510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204982.504514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204982.504516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.504518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204982.504521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.504524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204982.696458:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204982.696468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204982.696477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204982.696478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204982.696482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204982.696483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204982.696487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204982.696488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204982.696567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204982.696573:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204982.696577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204983.016484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204983.016492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204983.016494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204983.016499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204983.016502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204983.016506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204983.016508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204983.016510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204983.016512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204983.464454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204983.464461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204983.464463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204983.464468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204983.464471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204983.464475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204983.464478:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204983.464480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204983.464482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204983.528460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204983.528460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204983.528466:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204983.528467:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204983.528471:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204983.528471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204983.528476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204983.528481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204983.528482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204983.528486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204983.528487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204983.528488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204983.528489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204983.528492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204983.528494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204983.528496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204983.528499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204983.528501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204983.720525:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204983.720527:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204983.720529:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720532:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204983.720534:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720535:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720537:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204983.720540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204983.720549:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204983.720550:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204983.720551:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720552:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204983.720553:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720554:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720555:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204983.720559:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204983.720559:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204983.720560:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720561:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204983.720562:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720563:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204983.720564:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204983.720577:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204983.720579:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204983.720580:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204984.040472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204984.040477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204984.040479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204984.040482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204984.040485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204984.040488:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204984.040490:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204984.040491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204984.040493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204984.488478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204984.488485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204984.488487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204984.488492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204984.488495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204984.488498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204984.488499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204984.488501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204984.488503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204984.552471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204984.552478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204984.552480:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204984.552483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204984.552485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204984.552488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204984.552490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204984.552492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204984.552497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204984.552499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204984.552502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204984.552504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204984.552505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204984.552508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204984.552509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204984.552514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204984.552518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204984.552522:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204984.744494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204984.744500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204984.744504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204984.744505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204984.744508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204984.744509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204984.744512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204984.744513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204984.744567:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204984.744571:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204984.744574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204985.064436:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204985.064446:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301111810, at 4301112320 00000100:00000001:3.0:1788204985.064447:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204985.064451:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204985.064457:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204985.064460:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:3.0:1788204985.064462:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204985.064468:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204985.064470:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301111810, at 4301112320 00000100:00000001:3.0:1788204985.064473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204985.064478:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788204985.064483:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000001:3.0:1788204985.064484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204985.512489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204985.512496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204985.512498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204985.512504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204985.512508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204985.512523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204985.512526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204985.512529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204985.512532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204985.576448:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204985.576458:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204985.576462:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204985.576466:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204985.576470:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204985.576470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204985.576472:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204985.576479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204985.576480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204985.576484:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6440 00000100:00000001:0.0:1788204985.576486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.576486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204985.576496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204985.576496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000400:00000200:2.0:1788204985.576504:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788204985.576508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204985.576508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204985.576515:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204985.576515:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204985.576515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204985.576524:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788204985.576526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204985.576534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204985.576540:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204985.576543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.576544:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204985.576545:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204985.576551:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204985.576553:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204985.576556:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576557:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204985.576562:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204985.576565:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204985.576567:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204985.576571:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204985.576573:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204985.576575:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204985.576579:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576580:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204985.576583:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576585:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204985.576590:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204985.576592:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576594:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204985.576597:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204985.576600:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204985.576610:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576612:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204985.576614:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204985.576624:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788204985.576627:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204985.576632:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204985.576634:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576636:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204985.576638:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204985.576643:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204985.576650:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788204985.576656:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204985.576664:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204985.576672:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204985.576676:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204985.576679:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204985.576698:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204985.576701:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204985.576705:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204985.576707:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204985.576709:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204985.576712:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204985.576716:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6440 00000400:00000200:2.0:1788204985.576721:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204985.576721:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204985.576728:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1788204985.576733:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204985.576733:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788204985.576742:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204985.576751:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576753:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204985.576755:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204985.576756:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204985.576757:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204985.576765:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204985.576768:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204985.576784:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204985.576788:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204985.576796:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204985.576800:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788204985.576801:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204985.576803:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204985.576809:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576811:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788204985.576811:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1788204985.576822:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788204985.576822:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204985.576835:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788204985.576835:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204985.576847:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788204985.576850:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204985.576851:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204985.576854:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788204985.576855:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205917312:192.168.201.151@tcp:250:kworker.0 02000000:00000001:2.0:1788204985.576861:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204985.576862:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204985.576869:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576872:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204985.576873:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204985.576883:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204985.576886:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204985.576886:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788204985.576895:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576897:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204985.576901:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000010:3.0:1788204985.576901:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000020:00000040:2.0:1788204985.576910:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788204985.576912:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204985.576914:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204985.576917:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.576919:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204985.576921:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204985.576923:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204985.576926:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000400:00000200:3.0:1788204985.576927:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788204985.576931:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204985.576935:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00100000:2.0:1788204985.576942:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204985.576947:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:2.0:1788204985.576949:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204985.576954:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204985.576957:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204985.576959:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917312, portal 25 00000100:00000001:3.0:1788204985.576968:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204985.576974:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788204985.576976:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204985.576980:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788204985.576985:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204985.576986:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204985.576993:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204985.577000:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204985.577002:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788204985.577004:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204985.577008:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000001:2.0:1788204985.577010:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788204985.577013:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205917312, offset 0 00000100:00080000:2.0:1788204985.577014:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6440 00000400:00000200:2.0:1788204985.577023:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.577028:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788204985.577034:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204985.577036:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204985.577039:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577041:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204985.577042:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577044:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204985.577045:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204985.577046:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204985.577047:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204985.577048:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788204985.577049:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:2.0:1788204985.577054:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204985.577058:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788204985.577065:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204985.577066:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:2.0:1788204985.577075:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204985.577079:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788204985.577085:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204985.577091:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:2.0:1788204985.577092:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204985.577100:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788204985.577103:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788204985.577104:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577108:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.577111:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:2.0:1788204985.577112:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204985.577116:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204985.577120:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59022:59022:256:4294967295] 192.168.201.151@tcp LPNI seq info [59002:59002:8:4294967295] 02000000:00000001:2.0:1788204985.577124:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.577132:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204985.577136:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788204985.577140:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788204985.577150:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.577150:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788204985.577159:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204985.577164:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000010:2.0:1788204985.577170:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000800:00000010:3.0:1788204985.577171:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000020:00000040:2.0:1788204985.577177:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788204985.577184:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.0:1788204985.577185:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788204985.577186:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.577190:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204985.577194:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204985.577198:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204985.577199:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 02000000:00000010:2.0:1788204985.577207:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204985.577215:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204985.577218:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:2.0:1788204985.577224:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204985.577225:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204985.577243:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.577248:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000001:2.0:1788204985.577250:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204985.577254:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.577256:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788204985.577263:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204985.577270:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204985.577292:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000001:2.0:1788204985.577293:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204985.577297:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204985.577299:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00080000:2.0:1788204985.577301:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204985.577308:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204985.577310:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204985.577311:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204985.577313:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204985.577321:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6440 00000100:00000200:3.0:1788204985.577328:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204985.577330:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204985.577337:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204985.577343:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204985.577351:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577357:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204985.577384:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577387:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204985.577389:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204985.577390:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788204985.577392:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204985.577393:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788204985.577395:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204985.577399:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788204985.577403:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204985.577404:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577409:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204985.577414:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204985.577417:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204985.577424:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204985.577432:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204985.577444:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204985.577446:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204985.577450:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204985.577451:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204985.577455:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204985.577459:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204985.577459:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204985.577464:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.577469:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204985.577470:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788204985.577472:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788204985.577474:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301116930 00000008:00000001:2.0:1788204985.577479:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.577481:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204985.577483:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f40c0. 00000100:00000010:2.0:1788204985.577486:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788204985.577490:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788204985.577492:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204985.577493:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788204985.577495:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204985.577498:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204985.577499:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204985.577500:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301116930 02000000:00000001:2.0:1788204985.577501:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204985.577514:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000010:3.0:1788204985.577516:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4de0. 00000100:00000001:2.0:1788204985.577532:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.577532:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204985.577545:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788204985.577547:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204985.577549:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.577556:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204985.577566:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204985/real 1788204985] req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204985.577568:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204985.577575:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204985.577588:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204985.577595:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204985.577601:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788204985.577605:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204985.577606:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204985.577617:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000040:2.0:1788204985.577620:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6450) 00000400:00000010:3.0:1788204985.577621:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204985.577632:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204985.577637:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204985.577644:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577649:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204985.577651:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577652:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.577654:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204985.577656:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577661:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204985.577664:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577669:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204985.577671:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577672:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.577673:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577675:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204985.577677:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204985.577680:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204985.577683:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204985.577684:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204985.577687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204985.577692:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205917312:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204985.577695:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577697:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.577699:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205917312/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577703:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204985.577705:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204985.577707:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204985.577709:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204985.577712:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204985.577713:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204985.577715:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204985.577717:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:3.0:1788204985.577720:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204985.577721:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204985.577723:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788204985.577725:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204985.577726:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577728:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577733:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204985.577735:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.577736:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204985.577738:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204985.577739:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204985.577742:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577747:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204985.577748:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.577749:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204985.577751:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204985.577752:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204985.577754:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917440:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204985.577757:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204985.577758:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204985.577759:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.577761:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204985.577763:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204985.577765:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.577768:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204985.577770:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204985.577772:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204985.577774:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917440, portal 10 00000100:00000001:3.0:1788204985.577790:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204985.577790:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788204985.577796:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577800:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204985.577802:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204985.577804:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204985.577806:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205917440, offset 0 00000400:00000200:3.0:1788204985.577808:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204985.577812:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.577815:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204985.577817:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204985.577819:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204985.577821:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204985.577823:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204985.577825:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59023:59023:256:4294967295] 192.168.201.151@tcp LPNI seq info [59003:59003:8:4294967295] 00000400:00000200:3.0:1788204985.577828:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.577833:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204985.577836:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.577839:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204985.577842:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204985.577844:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204985.577848:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204985.577850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204985.577853:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.577855:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204985.577857:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204985.577859:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204985.577861:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204985.577863:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204985.577864:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204985.577867:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577872:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.577874:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577878:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.577879:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204985.577881:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204985.577884:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204985.577886:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204985.577887:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301112987 00000100:00000001:3.0:1788204985.577889:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.577891:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301112987 00000100:00000001:3.0:1788204985.577892:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.577894:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204985.577895:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.577897:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204985.577899:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204985/real 1788204985] req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204985.577906:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204985.577908:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204985.577909:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204985.577912:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204985.577913:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204985.577916:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204985.577919:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204985.577928:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577935:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204985.577937:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577939:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.577941:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204985.577945:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577952:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204985.577957:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577964:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204985.577965:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577967:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.577968:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.577969:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204985.577971:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204985.577973:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204985.577975:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204985.577977:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204985.577980:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204985.577985:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917440:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204985.577987:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204985.577990:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.577994:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204985.577995:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.577997:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204985.577998:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204985.578000:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204985.578002:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917568:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204985.578004:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204985.578005:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204985.578006:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.578007:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204985.578010:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204985.578011:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.578014:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204985.578016:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204985.578018:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204985.578021:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917568, portal 4 00000100:00000001:3.0:1788204985.578022:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204985.578023:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:3.0:1788204985.578027:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578035:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204985.578037:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204985.578038:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204985.578040:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205917568, offset 0 00000400:00000200:3.0:1788204985.578042:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204985.578045:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.578047:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204985.578049:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204985.578051:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204985.578053:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204985.578054:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204985.578056:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59024:59024:256:4294967295] 192.168.201.151@tcp LPNI seq info [59004:59004:8:4294967295] 00000400:00000200:3.0:1788204985.578059:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.578063:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204985.578066:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.578068:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204985.578071:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204985.578073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204985.578076:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204985.578078:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204985.578080:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.578082:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204985.578084:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204985.578086:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204985.578088:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204985.578089:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204985.578091:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204985.578093:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578098:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.578100:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578110:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.578111:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204985.578113:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204985.578115:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204985.578117:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204985.578118:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301112873 00000100:00000001:3.0:1788204985.578120:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.578121:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301112873 00000100:00000001:3.0:1788204985.578123:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.578124:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204985.578125:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.578126:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204985.578128:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204985/real 1788204985] req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204985.578133:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204985.578134:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204985.578135:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204985.578136:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204985.578137:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204985.578139:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204985.578141:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204985.578146:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578149:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204985.578150:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578151:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.578152:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204985.578154:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578159:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204985.578162:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578166:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204985.578167:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578168:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.578169:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578170:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204985.578172:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204985.578173:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204985.578175:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204985.578176:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204985.578179:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204985.578183:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917568:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204985.578185:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204985.578188:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578192:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204985.578193:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.578195:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204985.578196:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204985.578197:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204985.578199:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917696:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204985.578201:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204985.578202:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204985.578203:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204985.578205:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204985.578207:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204985.578208:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204985.578210:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204985.578211:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204985.578213:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204985.578215:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917696, portal 4 00000100:00000001:3.0:1788204985.578216:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204985.578217:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204985.578221:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578225:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204985.578226:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204985.578227:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204985.578229:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205917696, offset 0 00000400:00000200:3.0:1788204985.578231:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204985.578233:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.578234:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204985.578236:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204985.578238:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204985.578239:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204985.578241:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204985.578243:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59025:59025:256:4294967295] 192.168.201.151@tcp LPNI seq info [59005:59005:8:4294967295] 00000400:00000200:3.0:1788204985.578246:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204985.578250:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204985.578252:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.578254:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204985.578256:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204985.578258:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204985.578260:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204985.578262:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204985.578264:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204985.578266:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204985.578268:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204985.578270:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204985.578272:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204985.578273:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204985.578275:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204985.578277:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578281:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.578283:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578287:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.578288:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204985.578290:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204985.578292:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204985.578294:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204985.578295:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301112879 00000100:00000001:3.0:1788204985.578297:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.578298:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301112879 00000100:00000001:3.0:1788204985.578300:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301116930 00000100:00000001:3.0:1788204985.578301:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204985.578302:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.578303:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204985.578305:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204985/real 1788204985] req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204985.578310:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204985.578310:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204985.578311:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204985.578313:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204985.578314:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204985.578315:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204985.578317:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204985.578323:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578326:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204985.578327:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578328:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.578329:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204985.578331:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578335:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204985.578337:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578341:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204985.578342:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578343:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204985.578344:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578345:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204985.578346:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204985.578348:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204985.578350:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204985.578351:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204985.578353:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204985.578358:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917696:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204985.578360:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578361:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.578382:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205917440/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578387:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204985.578388:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204985.578390:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204985.578391:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204985.578392:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204985.578393:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204985.578395:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204985.578396:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788204985.578398:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204985.578399:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204985.578400:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788204985.578402:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204985.578403:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578404:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.578406:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205917568/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578411:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204985.578412:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204985.578412:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204985.578414:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204985.578414:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204985.578415:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204985.578416:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204985.578417:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204985.578418:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204985.578419:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204985.578420:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788204985.578421:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204985.578422:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578423:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204985.578424:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205917696/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204985.578428:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204985.578429:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204985.578430:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204985.578432:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204985.578433:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204985.578434:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204985.578435:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204985.578436:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788204985.578437:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204985.578437:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204985.578438:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204985.578440:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204985.578440:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578441:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204985.578444:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204985.578445:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204985.578446:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204985.578447:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204985.768542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204985.768549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204985.768554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204985.768555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204985.768558:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204985.768559:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204985.768562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204985.768563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204985.768579:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204985.768581:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204985.768583:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204986.088485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204986.088493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204986.088496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204986.088501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204986.088506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204986.088510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204986.088513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204986.088516:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204986.088519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204986.536492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204986.536498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204986.536500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204986.536504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204986.536508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204986.536512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204986.536515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204986.536517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204986.536520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204986.600489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204986.600489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204986.600498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204986.600498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204986.600502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204986.600503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204986.600511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204986.600512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204986.600518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204986.600519:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204986.600525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204986.600526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204986.600532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204986.600536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204986.600540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204986.600541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204986.600545:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204986.600548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204986.792511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204986.792518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204986.792523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204986.792525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204986.792527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204986.792528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204986.792531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204986.792532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204986.792584:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204986.792588:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204986.792591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204987.112437:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204987.112447:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204987.112450:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.112457:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204987.112461:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204987.112465:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.112470:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.112473:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.112476:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204987.560536:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204987.560546:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204987.560549:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204987.560556:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204987.560561:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204987.560566:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204987.560578:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204987.560581:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204987.560584:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204987.624484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204987.624486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204987.624495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204987.624495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204987.624500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.624500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204987.624508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204987.624508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204987.624514:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204987.624515:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204987.624518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204987.624520:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.624521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.624525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.624528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204987.624530:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204987.624533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204987.624535:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204987.816498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204987.816509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204987.816516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204987.816517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204987.816521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204987.816522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204987.816526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204987.816527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204987.816551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204987.816553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204987.816555:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204988.136497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204988.136505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204988.136507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204988.136512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204988.136515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204988.136519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.136521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204988.136524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.136526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204988.584461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204988.584471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204988.584474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204988.584481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204988.584486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204988.584491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204988.584494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204988.584497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204988.584501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204988.648484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204988.648484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204988.648493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204988.648493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204988.648496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204988.648497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204988.648504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204988.648504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204988.648509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204988.648510:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204988.648513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204988.648514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.648517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204988.648520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.648523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204988.648526:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204988.648530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204988.648533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204988.840498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204988.840508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204988.840516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204988.840517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204988.840521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204988.840523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204988.840527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204988.840529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204988.840581:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204988.840586:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204988.840590:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204989.160489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204989.160497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204989.160499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204989.160504:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204989.160507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204989.160511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204989.160513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204989.160515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204989.160518:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204989.608480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204989.608488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204989.608490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204989.608495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204989.608499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204989.608503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204989.608505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204989.608507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204989.608509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204989.672449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204989.672455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204989.672458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204989.672460:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204989.672462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204989.672464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204989.672469:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788204989.672470:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204989.672476:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204989.672476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204989.672483:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204989.672483:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204989.672489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204989.672491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204989.672494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204989.672496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204989.672499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204989.672502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204989.864513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204989.864520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204989.864525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204989.864526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204989.864529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204989.864530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204989.864534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204989.864535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204989.864556:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204989.864559:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204989.864562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788204990.184483:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301116930, at 4301117440 00000100:00000001:3.0:1788204990.184483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788204990.184488:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301116930, at 4301117440 00000100:00000001:3.0:1788204990.184489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204990.184492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204990.184500:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00100000:3.0:1788204990.184504:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788204990.184508:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000001:3.0:1788204990.184508:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204990.184518:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.184522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.184523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.184526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204990.632476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204990.632484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204990.632486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204990.632491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204990.632494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204990.632498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204990.632500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204990.632502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204990.632504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204990.696455:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788204990.696458:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204990.696458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204990.696465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788204990.696466:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204990.696466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204990.696470:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.696470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696471:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788204990.696476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204990.696477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788204990.696478:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204990.696480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204990.696484:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204990.696485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204990.696485:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788204990.696490:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.696490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204990.696492:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6445 00000100:00000001:0.0:1788204990.696494:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204990.696498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204990.696502:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204990.696502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.696505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.696509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204990.696510:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204990.696513:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204990.696516:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696518:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.696520:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696522:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204990.696525:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204990.696527:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788204990.696528:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204990.696531:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788204990.696532:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788204990.696533:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788204990.696535:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696536:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.696537:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696539:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204990.696541:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788204990.696543:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696544:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204990.696546:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204990.696548:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204990.696550:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696551:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204990.696553:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204990.696559:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788204990.696562:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788204990.696565:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204990.696566:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696568:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204990.696570:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204990.696573:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204990.696579:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788204990.696584:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204990.696591:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204990.696597:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204990.696600:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204990.696603:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204990.696619:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204990.696621:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204990.696623:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204990.696624:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204990.696625:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204990.696627:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204990.696629:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6445 00000400:00000200:2.0:1788204990.696631:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204990.696633:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204990.696635:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204990.696637:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204990.696638:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204990.696639:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.696642:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788204990.696642:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204990.696644:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204990.696645:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204990.696646:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788204990.696648:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788204990.696650:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204990.696652:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788204990.696654:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788204990.696655:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204990.696656:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788204990.696661:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696664:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.696668:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696673:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204990.696675:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788204990.696677:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788204990.696679:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204990.696682:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788204990.696685:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204990.696701:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204990.696703:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204990.696707:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205917824:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788204990.696710:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204990.696713:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.696713:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204990.696716:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204990.696717:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204990.696718:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204990.696722:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1788204990.696723:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:2.0:1788204990.696726:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.696727:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696731:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204990.696732:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788204990.696736:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788204990.696737:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000008:00000001:2.0:1788204990.696741:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696743:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204990.696744:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000010:2.0:1788204990.696747:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788204990.696750:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000200:3.0:1788204990.696750:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917824, portal 25 02000000:00000001:2.0:1788204990.696752:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204990.696754:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.696754:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204990.696757:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204990.696757:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 02000000:00000001:2.0:1788204990.696760:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204990.696763:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204990.696765:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204990.696767:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788204990.696771:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.696775:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788204990.696779:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204990.696779:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204990.696784:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000001:2.0:1788204990.696789:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204990.696790:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205917824, offset 0 00000100:00000001:2.0:1788204990.696791:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204990.696797:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204990.696803:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696806:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204990.696810:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204990.696814:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788204990.696816:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204990.696819:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204990.696820:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204990.696824:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:2.0:1788204990.696825:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204990.696831:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6445 00000400:00000200:3.0:1788204990.696832:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204990.696839:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204990.696843:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204990.696846:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204990.696849:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59026:59026:256:4294967295] 192.168.201.151@tcp LPNI seq info [59006:59006:8:4294967295] 00000400:00000200:3.0:1788204990.696852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204990.696856:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788204990.696863:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204990.696864:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788204990.696866:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204990.696871:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696873:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204990.696874:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788204990.696876:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696879:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788204990.696880:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000010:3.0:1788204990.696880:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000020:00000040:2.0:1788204990.696882:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788204990.696885:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788204990.696886:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788204990.696888:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788204990.696890:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.0:1788204990.696890:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:2.0:1788204990.696895:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788204990.696896:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788204990.696898:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.696902:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696906:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.1:1788204990.696906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:2.0:1788204990.696908:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.1:1788204990.696910:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00080000:2.0:1788204990.696912:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788204990.696916:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204990.696917:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788204990.696918:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1788204990.696921:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 02000000:00000001:2.0:1788204990.696922:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204990.696925:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204990.696926:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204990.696927:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204990.696930:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788204990.696932:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000008:00000001:2.0:1788204990.696934:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696936:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204990.696937:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000010:2.0:1788204990.696939:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000400:00000010:3.0:1788204990.696940:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000020:00000040:2.0:1788204990.696943:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:3.0:1788204990.696945:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788204990.696946:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204990.696948:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696950:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204990.696952:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1788204990.696952:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204990.696954:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204990.696959:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788204990.696963:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204990.696967:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204990.696968:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.696974:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204990.696976:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204990.696980:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.696985:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204990.696986:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.696987:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204990.696993:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.696994:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204990.696997:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204990.697000:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788204990.697001:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204990.697005:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204990.697005:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301122049 00000100:00080000:2.0:1788204990.697009:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1788204990.697011:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 00000100:00000001:2.0:1788204990.697013:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204990.697017:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6445 00000100:00000001:3.0:1788204990.697018:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000400:00000200:2.0:1788204990.697025:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204990.697025:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301122049 00000100:00000010:3.0:1788204990.697030:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f48a0. 00000400:00000200:2.0:1788204990.697032:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204990.697034:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788204990.697038:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204990.697040:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697041:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788204990.697044:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204990.697045:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204990/real 1788204990] req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204990.697051:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.697056:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.697061:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.697066:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000001:3.0:1788204990.697066:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204990.697068:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788204990.697071:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:3.0:1788204990.697071:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000020:00000040:2.0:1788204990.697074:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000400:00000200:3.0:1788204990.697075:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000020:00000001:2.0:1788204990.697079:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:3.0:1788204990.697079:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204990.697083:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1788204990.697084:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000200:3.0:1788204990.697087:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204990.697088:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788204990.697094:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788204990.697101:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:3.0:1788204990.697102:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204990.697103:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204990.697108:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204990.697111:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788204990.697114:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788204990.697114:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204990.697117:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204990.697118:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788204990.697120:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788204990.697121:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697122:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204990.697126:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204990.697129:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204990.697133:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204990.697134:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204990.697137:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.697141:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:3.0:1788204990.697141:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788204990.697144:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788204990.697148:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.697151:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697155:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:2.0:1788204990.697157:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204990.697158:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788204990.697160:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788204990.697161:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697164:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204990.697165:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204990.697167:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204990.697169:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204990.697170:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204990.697172:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:3.0:1788204990.697172:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788204990.697175:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204990.697178:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204990.697178:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204990.697182:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:2.0:1788204990.697184:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000040:3.0:1788204990.697186:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204990.697188:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204990.697195:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204990.697199:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205917824:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788204990.697204:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204990.697206:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204990.697207:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697209:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697213:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205917824/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204990.697216:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697221:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204990.697223:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697226:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204990.697231:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788204990.697232:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204990.697233:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204990.697235:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204990.697237:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:2.0:1788204990.697239:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6455) 02000000:00000010:3.0:1788204990.697239:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:3.0:1788204990.697244:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204990.697245:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204990.697246:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788204990.697248:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204990.697251:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697252:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697258:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204990.697260:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697263:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204990.697264:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204990.697266:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204990.697270:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697281:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.697283:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697285:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204990.697286:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204990.697289:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204990.697291:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917952:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204990.697295:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204990.697297:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204990.697298:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697300:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697303:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204990.697304:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204990.697307:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204990.697309:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204990.697311:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204990.697313:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205917952, portal 10 00000100:00000001:3.0:1788204990.697315:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204990.697315:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788204990.697320:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697324:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204990.697326:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204990.697328:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204990.697330:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205917952, offset 0 00000400:00000200:3.0:1788204990.697334:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697338:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697341:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204990.697344:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204990.697347:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204990.697349:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204990.697351:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697354:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59027:59027:256:4294967295] 192.168.201.151@tcp LPNI seq info [59007:59007:8:4294967295] 00000400:00000200:3.0:1788204990.697358:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697380:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204990.697384:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697386:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204990.697389:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204990.697391:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204990.697393:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204990.697395:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204990.697397:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697398:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204990.697400:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204990.697402:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204990.697404:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204990.697405:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204990.697407:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204990.697409:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697414:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697416:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697420:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697420:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204990.697422:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204990.697425:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204990.697427:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204990.697428:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301118107 00000100:00000001:3.0:1788204990.697430:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697431:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301118107 00000100:00000001:3.0:1788204990.697432:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697433:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204990.697434:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697436:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204990.697437:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204990/real 1788204990] req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204990.697442:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204990.697442:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204990.697444:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204990.697445:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204990.697446:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204990.697447:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204990.697449:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204990.697453:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697456:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204990.697457:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697458:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697459:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204990.697461:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697464:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204990.697466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697470:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204990.697472:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697472:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697473:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697474:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204990.697475:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204990.697477:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204990.697478:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204990.697479:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204990.697481:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204990.697486:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205917952:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204990.697488:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204990.697490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697494:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.697495:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697496:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204990.697497:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204990.697498:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204990.697499:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918080:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204990.697501:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204990.697502:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204990.697503:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697504:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697506:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204990.697507:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204990.697509:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204990.697510:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204990.697512:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204990.697514:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918080, portal 4 00000100:00000001:3.0:1788204990.697515:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204990.697516:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:3.0:1788204990.697519:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697522:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204990.697524:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204990.697525:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204990.697526:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205918080, offset 0 00000400:00000200:3.0:1788204990.697543:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697548:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204990.697550:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204990.697552:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204990.697555:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204990.697557:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697559:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59028:59028:256:4294967295] 192.168.201.151@tcp LPNI seq info [59008:59008:8:4294967295] 00000400:00000200:3.0:1788204990.697562:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697566:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204990.697569:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697571:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204990.697573:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204990.697575:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204990.697577:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204990.697579:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204990.697581:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697582:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204990.697584:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204990.697586:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204990.697587:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204990.697589:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204990.697590:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204990.697592:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697596:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697598:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697602:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697602:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204990.697604:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204990.697606:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204990.697607:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204990.697608:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301117993 00000100:00000001:3.0:1788204990.697610:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697611:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301117993 00000100:00000001:3.0:1788204990.697613:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697614:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204990.697615:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697616:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204990.697618:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204990/real 1788204990] req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204990.697622:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204990.697623:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204990.697624:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204990.697625:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204990.697625:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204990.697626:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204990.697628:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204990.697633:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697636:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204990.697637:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697638:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697639:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204990.697641:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697644:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204990.697646:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697652:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204990.697654:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697655:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697656:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697658:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204990.697660:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204990.697662:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204990.697665:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204990.697667:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204990.697671:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204990.697679:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918080:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204990.697682:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204990.697685:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697702:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.697703:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697705:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204990.697706:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204990.697708:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204990.697710:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918208:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204990.697712:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204990.697714:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204990.697715:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204990.697716:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697718:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204990.697720:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204990.697722:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204990.697723:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204990.697725:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204990.697727:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918208, portal 4 00000100:00000001:3.0:1788204990.697728:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204990.697729:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788204990.697732:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697737:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204990.697738:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204990.697739:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204990.697741:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205918208, offset 0 00000400:00000200:3.0:1788204990.697743:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697745:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697747:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204990.697749:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204990.697751:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204990.697752:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204990.697753:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204990.697755:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59029:59029:256:4294967295] 192.168.201.151@tcp LPNI seq info [59009:59009:8:4294967295] 00000400:00000200:3.0:1788204990.697758:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204990.697762:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204990.697765:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697766:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204990.697768:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204990.697770:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204990.697772:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204990.697774:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204990.697776:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204990.697777:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204990.697779:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204990.697781:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204990.697783:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204990.697784:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204990.697785:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204990.697788:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697794:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697797:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697808:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697810:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204990.697813:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204990.697815:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204990.697818:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204990.697820:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301117999 00000100:00000001:3.0:1788204990.697823:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697825:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301117999 00000100:00000001:3.0:1788204990.697827:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301122049 00000100:00000001:3.0:1788204990.697829:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204990.697830:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697832:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204990.697835:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204990/real 1788204990] req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204990.697840:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204990.697841:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204990.697842:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204990.697843:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204990.697844:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204990.697845:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204990.697847:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204990.697852:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697855:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204990.697856:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697857:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697858:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204990.697859:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697863:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204990.697866:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697869:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204990.697870:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697871:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204990.697872:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697873:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204990.697874:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204990.697875:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204990.697877:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204990.697878:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204990.697880:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204990.697884:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918208:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204990.697886:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697887:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697889:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205917952/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697893:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204990.697894:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697895:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204990.697896:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204990.697897:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204990.697898:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204990.697899:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204990.697900:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788204990.697902:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204990.697903:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204990.697904:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788204990.697905:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204990.697906:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697907:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697909:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205918080/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697913:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204990.697913:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697914:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204990.697915:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204990.697916:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204990.697917:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204990.697918:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204990.697918:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788204990.697919:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204990.697920:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204990.697921:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788204990.697922:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204990.697922:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697923:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204990.697925:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205918208/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204990.697928:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204990.697928:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204990.697929:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204990.697930:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204990.697931:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204990.697931:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204990.697932:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204990.697933:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788204990.697934:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204990.697934:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204990.697935:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788204990.697936:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204990.697937:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697938:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.697940:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204990.697941:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.697942:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204990.697943:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204990.888498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204990.888507:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204990.888509:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204990.888512:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888514:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204990.888516:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888518:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888519:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204990.888523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204990.888525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204990.888526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204990.888588:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204990.888594:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204990.888598:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204991.208465:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204991.208470:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204991.208471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204991.208475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204991.208477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204991.208481:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204991.208482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204991.208484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204991.208486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204991.656501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204991.656511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204991.656513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204991.656520:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204991.656524:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204991.656529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204991.656532:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204991.656535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204991.656539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204991.720460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204991.720468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204991.720471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204991.720476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204991.720477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204991.720480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204991.720482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204991.720483:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788204991.720485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204991.720489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204991.720489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204991.720491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204991.720494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204991.720498:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204991.720502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204991.720502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204991.720504:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204991.720508:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204991.912513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204991.912520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204991.912525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204991.912526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204991.912529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204991.912530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204991.912532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204991.912533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204991.912550:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204991.912552:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204991.912560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204992.232458:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204992.232466:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204992.232468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.232474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204992.232477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204992.232480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.232483:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.232485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.232488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204992.680451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204992.680457:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204992.680459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204992.680462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204992.680465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204992.680467:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204992.680469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204992.680470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204992.680472:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204992.744448:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204992.744453:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204992.744455:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204992.744457:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.744458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204992.744462:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204992.744463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204992.744468:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204992.744468:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204992.744471:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.744473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204992.744474:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204992.744478:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.744479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204992.744481:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.744483:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.744486:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.744489:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204992.936472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204992.936478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204992.936481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204992.936482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204992.936484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204992.936485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204992.936488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204992.936489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204992.936558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204992.936562:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204992.936566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204993.256460:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204993.256467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204993.256469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204993.256473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204993.256476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204993.256479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.256480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204993.256482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.256484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204993.704424:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204993.704430:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204993.704432:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204993.704435:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204993.704438:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204993.704440:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204993.704442:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204993.704444:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204993.704446:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204993.768460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204993.768466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204993.768467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204993.768472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204993.768475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204993.768477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.768479:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204993.768487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204993.768487:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.768489:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204993.768493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204993.768496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204993.768500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204993.768503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204993.768505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204993.768507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204993.768509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204993.768511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204993.960514:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204993.960516:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204993.960518:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960522:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204993.960523:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960524:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960526:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204993.960529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204993.960532:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204993.960533:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204993.960534:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960535:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204993.960535:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960536:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960537:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204993.960540:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788204993.960541:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788204993.960541:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960542:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788204993.960543:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960544:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204993.960545:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204993.960558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204993.960560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204993.960562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204994.280424:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204994.280429:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204994.280431:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204994.280435:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204994.280438:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204994.280441:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204994.280444:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204994.280446:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204994.280449:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204994.728458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204994.728463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204994.728465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204994.728468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204994.728470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204994.728473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204994.728474:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204994.728476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204994.728478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204994.792472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204994.792479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204994.792481:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204994.792485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204994.792488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204994.792491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204994.792493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204994.792495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204994.792496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204994.792498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204994.792500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204994.792503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204994.792505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204994.792508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204994.792511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204994.792513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204994.792514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204994.792517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204994.984464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204994.984469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204994.984472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204994.984473:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204994.984475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204994.984476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204994.984478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204994.984478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204994.984531:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204994.984535:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204994.984539:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788204995.304451:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301122049, at 4301122560 00000100:00000001:3.0:1788204995.304453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204995.304458:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788204995.304459:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301122049, at 4301122560 00000100:00000001:3.0:1788204995.304463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204995.304467:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00100000:3.0:1788204995.304469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788204995.304472:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:3.0:1788204995.304475:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204995.304479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.304483:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.304485:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.304489:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204995.752473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204995.752479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204995.752481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204995.752484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204995.752487:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204995.752490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204995.752492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204995.752493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204995.752495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.816434:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204995.816439:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204995.816441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204995.816444:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204995.816446:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788204995.816450:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204995.816452:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204995.816454:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204995.816459:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204995.816460:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204995.816465:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204995.816466:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6450 00000100:00000001:3.0:1788204995.816469:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204995.816471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204995.816472:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204995.816474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1788204995.816477:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204995.816477:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204995.816478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204995.816483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000400:00000200:0.0:1788204995.816486:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788204995.816487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788204995.816490:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788204995.816491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204995.816494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1788204995.816495:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788204995.816496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204995.816499:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788204995.816500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204995.816503:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816505:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204995.816508:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204995.816509:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788204995.816511:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204995.816514:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788204995.816515:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788204995.816517:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788204995.816520:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816521:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204995.816522:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816523:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204995.816526:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788204995.816527:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816529:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204995.816530:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204995.816532:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204995.816534:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.816535:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204995.816537:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204995.816545:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788204995.816547:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788204995.816550:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204995.816551:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.816553:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204995.816554:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204995.816557:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204995.816563:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788204995.816568:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204995.816582:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204995.816589:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204995.816592:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204995.816595:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.816621:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204995.816623:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204995.816625:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204995.816626:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204995.816627:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204995.816629:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204995.816630:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6450 00000400:00000200:0.0:1788204995.816633:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204995.816635:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204995.816637:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204995.816639:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816640:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204995.816642:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816644:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204995.816645:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204995.816646:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788204995.816647:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204995.816649:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788204995.816649:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788204995.816652:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204995.816653:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788204995.816654:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788204995.816656:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204995.816658:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816661:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204995.816664:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:3.0:1788204995.816664:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.816667:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788204995.816671:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788204995.816675:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816678:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.816680:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788204995.816681:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204995.816683:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204995.816687:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204995.816687:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204995.816691:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.816691:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204995.816694:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204995.816697:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205918336:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204995.816700:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204995.816702:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788204995.816704:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204995.816705:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1788204995.816709:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.816709:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.816714:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.816714:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204995.816720:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000010:0.0:1788204995.816721:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 02000000:00000001:3.0:1788204995.816724:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788204995.816729:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:3.0:1788204995.816733:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788204995.816735:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204995.816738:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204995.816739:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788204995.816741:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204995.816744:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204995.816747:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204995.816748:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 02000000:00000010:0.0:1788204995.816752:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000200:3.0:1788204995.816755:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918336, portal 25 00000100:00000001:0.0:1788204995.816757:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.816760:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204995.816762:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00100000:0.0:1788204995.816766:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204995.816772:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.816783:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204995.816785:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204995.816788:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204995.816790:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204995.816793:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000040:0.0:1788204995.816795:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204995.816797:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205918336, offset 0 00000100:00000001:0.0:1788204995.816806:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204995.816806:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788204995.816810:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204995.816812:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204995.816814:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204995.816814:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788204995.816816:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204995.816818:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204995.816821:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:0.0:1788204995.816822:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6450 00000400:00000200:3.0:1788204995.816829:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204995.816830:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204995.816835:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204995.816837:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:0.0:1788204995.816838:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204995.816841:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204995.816842:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816845:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204995.816845:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:0.0:1788204995.816850:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788204995.816852:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59030:59030:256:4294967295] 192.168.201.151@tcp LPNI seq info [59010:59010:8:4294967295] 00000100:00000001:0.0:1788204995.816853:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788204995.816857:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788204995.816859:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788204995.816859:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788204995.816861:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788204995.816864:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788204995.816865:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788204995.816868:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788204995.816871:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:0.0:1788204995.816875:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816878:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204995.816880:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788204995.816881:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816884:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000010:3.0:1788204995.816885:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000020:00000040:0.0:1788204995.816886:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788204995.816890:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204995.816892:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204995.816894:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788204995.816895:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204995.816899:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1788204995.816900:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 02000000:00000001:0.0:1788204995.816902:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.816905:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:3.1:1788204995.816906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00100000:0.0:1788204995.816907:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:3.1:1788204995.816910:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000008:00000001:0.0:1788204995.816911:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.816914:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788204995.816918:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000010:0.0:1788204995.816919:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000800:00000010:3.0:1788204995.816920:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000020:00000040:0.0:1788204995.816922:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788204995.816925:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204995.816926:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204995.816926:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788204995.816928:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204995.816930:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204995.816932:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1788204995.816933:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204995.816938:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000400:00000200:3.0:1788204995.816940:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000001:0.0:1788204995.816942:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204995.816944:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00100000:0.0:1788204995.816948:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204995.816951:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788204995.816956:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204995.816959:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204995.816960:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788204995.816963:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.816982:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204995.816983:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204995.816987:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788204995.816988:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204995.816997:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204995.817000:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.817001:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204995.817003:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788204995.817005:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204995.817009:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204995.817010:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204995.817016:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6450 00000400:00000200:0.0:1788204995.817020:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817024:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788204995.817025:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204995.817028:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301127169 00000100:00080000:0.0:1788204995.817029:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1788204995.817034:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4000. 00000100:00000040:0.0:1788204995.817035:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.817040:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204995.817041:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788204995.817043:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788204995.817047:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788204995.817048:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301127169 00000020:00000001:0.0:1788204995.817052:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000010:3.0:1788204995.817053:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f41e0. 00000020:00000040:0.0:1788204995.817056:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788204995.817057:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788204995.817061:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788204995.817062:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788204995.817066:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788204995.817066:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817068:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788204995.817071:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00100000:3.0:1788204995.817074:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204995/real 1788204995] req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788204995.817075:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788204995.817084:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000001:3.0:1788204995.817085:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788204995.817086:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:3.0:1788204995.817087:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1788204995.817090:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:3.0:1788204995.817091:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788204995.817094:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788204995.817095:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000020:00000040:0.0:1788204995.817098:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000010:3.0:1788204995.817099:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204995.817104:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1788204995.817105:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788204995.817109:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204995.817109:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204995.817113:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204995.817116:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204995.817120:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204995.817120:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.817124:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204995.817126:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788204995.817129:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:0.0:1788204995.817130:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817132:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204995.817133:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817137:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:0.0:1788204995.817138:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204995.817139:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788204995.817143:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00100000:3.0:1788204995.817144:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204995.817146:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204995.817148:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204995.817151:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204995.817154:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204995.817157:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204995.817160:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204995.817163:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817164:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204995.817167:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1788204995.817168:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:3.0:1788204995.817168:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817170:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817173:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204995.817175:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817175:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204995.817181:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788204995.817186:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204995.817190:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204995.817193:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204995.817199:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.817204:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788204995.817208:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205918336:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788204995.817209:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817213:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817215:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204995.817218:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204995.817220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205918336/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204995.817239:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817240:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1788204995.817244:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6460) 02000000:00000001:3.0:1788204995.817244:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204995.817246:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204995.817250:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204995.817251:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204995.817252:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788204995.817253:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204995.817255:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:3.0:1788204995.817258:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204995.817259:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204995.817260:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788204995.817262:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204995.817263:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817265:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817270:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204995.817271:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817273:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204995.817274:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204995.817275:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204995.817277:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817282:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.817283:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817284:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204995.817285:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204995.817286:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204995.817288:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918464:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204995.817290:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204995.817291:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204995.817292:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817293:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204995.817295:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204995.817296:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204995.817299:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204995.817300:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204995.817302:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204995.817304:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918464, portal 10 00000100:00000001:3.0:1788204995.817305:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204995.817306:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788204995.817310:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817314:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204995.817316:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204995.817317:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204995.817319:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205918464, offset 0 00000400:00000200:3.0:1788204995.817321:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817323:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817325:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204995.817327:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204995.817329:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204995.817330:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204995.817337:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817339:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59031:59031:256:4294967295] 192.168.201.151@tcp LPNI seq info [59011:59011:8:4294967295] 00000400:00000200:3.0:1788204995.817342:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817347:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204995.817351:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817354:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204995.817358:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204995.817361:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204995.817387:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204995.817390:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204995.817394:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817396:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204995.817400:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204995.817404:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204995.817407:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204995.817409:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204995.817411:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204995.817415:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817428:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.817431:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817435:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817436:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204995.817439:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204995.817442:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788204995.817444:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204995.817452:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301123227 00000100:00000001:3.0:1788204995.817454:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817455:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301123227 00000100:00000001:3.0:1788204995.817457:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817458:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204995.817459:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817460:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204995.817463:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204995/real 1788204995] req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204995.817468:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204995.817468:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204995.817470:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204995.817471:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204995.817472:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204995.817473:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204995.817475:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204995.817480:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817483:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204995.817485:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817486:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817487:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204995.817489:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817492:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204995.817494:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817498:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204995.817499:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817500:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817500:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817501:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204995.817503:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204995.817504:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204995.817506:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204995.817507:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204995.817509:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204995.817513:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918464:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788204995.817515:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204995.817517:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817521:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.817522:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817523:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204995.817524:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204995.817525:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204995.817527:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204995.817529:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204995.817530:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204995.817531:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817532:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204995.817534:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204995.817535:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204995.817537:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204995.817539:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204995.817540:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204995.817542:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918592, portal 4 00000100:00000001:3.0:1788204995.817543:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204995.817544:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788204995.817547:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817551:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204995.817553:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204995.817554:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204995.817555:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205918592, offset 0 00000400:00000200:3.0:1788204995.817557:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817560:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817563:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204995.817571:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204995.817573:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204995.817575:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204995.817578:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817580:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59032:59032:256:4294967295] 192.168.201.151@tcp LPNI seq info [59012:59012:8:4294967295] 00000400:00000200:3.0:1788204995.817584:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817590:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204995.817595:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817609:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204995.817614:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204995.817616:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204995.817620:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204995.817633:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204995.817635:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817637:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204995.817639:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204995.817641:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204995.817643:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204995.817644:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204995.817646:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204995.817648:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817653:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.817655:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817659:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817660:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204995.817662:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204995.817664:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788204995.817666:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204995.817667:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301123113 00000100:00000001:3.0:1788204995.817669:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817670:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301123113 00000100:00000001:3.0:1788204995.817671:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817672:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204995.817673:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817675:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204995.817677:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204995/real 1788204995] req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204995.817681:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204995.817682:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204995.817683:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204995.817684:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204995.817685:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204995.817686:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204995.817688:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204995.817692:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817695:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204995.817696:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817697:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817698:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204995.817700:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817703:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204995.817705:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817709:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204995.817710:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817711:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817712:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817713:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204995.817714:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204995.817716:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204995.817717:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204995.817719:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204995.817721:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204995.817730:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204995.817734:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204995.817737:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817743:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.817744:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817746:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204995.817748:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204995.817750:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204995.817752:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918720:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204995.817756:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204995.817757:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204995.817759:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204995.817760:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204995.817764:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204995.817766:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204995.817769:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788204995.817771:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788204995.817774:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788204995.817776:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918720, portal 4 00000100:00000001:3.0:1788204995.817777:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204995.817778:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788204995.817782:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817785:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204995.817787:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788204995.817788:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788204995.817789:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205918720, offset 0 00000400:00000200:3.0:1788204995.817792:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817794:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817796:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788204995.817806:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204995.817808:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788204995.817811:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204995.817816:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788204995.817819:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59033:59033:256:4294967295] 192.168.201.151@tcp LPNI seq info [59013:59013:8:4294967295] 00000400:00000200:3.0:1788204995.817823:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788204995.817831:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788204995.817834:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817838:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788204995.817841:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788204995.817843:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204995.817847:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788204995.817849:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204995.817852:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788204995.817854:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788204995.817856:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204995.817859:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204995.817861:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788204995.817862:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788204995.817864:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204995.817868:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817875:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.817877:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817883:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817884:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204995.817887:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204995.817889:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788204995.817892:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788204995.817893:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301123119 00000100:00000001:3.0:1788204995.817895:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817897:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301123119 00000100:00000001:3.0:1788204995.817899:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301127169 00000100:00000001:3.0:1788204995.817900:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204995.817902:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.817903:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204995.817906:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204995/real 1788204995] req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204995.817912:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204995.817913:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204995.817914:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788204995.817917:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788204995.817918:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788204995.817920:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204995.817922:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204995.817928:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817933:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204995.817935:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817936:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817938:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204995.817941:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817946:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204995.817949:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817955:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204995.817957:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817959:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204995.817960:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817961:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204995.817963:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204995.817965:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204995.817968:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788204995.817969:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204995.817972:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204995.817980:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918720:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788204995.817983:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.817985:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.817988:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205918464/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.817994:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204995.817996:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204995.817997:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788204995.817999:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204995.818000:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204995.818001:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788204995.818002:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204995.818003:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788204995.818005:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204995.818006:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204995.818007:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788204995.818009:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204995.818010:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.818011:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.818013:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205918592/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.818020:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204995.818021:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204995.818022:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788204995.818023:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204995.818023:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204995.818024:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788204995.818025:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204995.818026:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788204995.818027:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204995.818027:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204995.818028:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788204995.818029:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204995.818030:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.818031:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204995.818032:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205918720/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204995.818035:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204995.818036:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204995.818037:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788204995.818038:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204995.818038:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788204995.818039:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788204995.818040:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788204995.818041:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788204995.818041:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204995.818042:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204995.818043:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788204995.818044:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204995.818045:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.818046:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.818048:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204995.818049:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.818050:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204995.818050:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204996.008509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204996.008514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204996.008520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204996.008521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204996.008524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204996.008526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204996.008529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204996.008530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204996.008548:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204996.008550:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204996.008553:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204996.328428:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204996.328434:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204996.328440:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204996.328444:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204996.328447:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204996.328451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.328453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204996.328455:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.328456:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204996.776445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204996.776451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204996.776453:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204996.776462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204996.776468:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204996.776471:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204996.776472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204996.776474:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204996.776476:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204996.840441:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204996.840447:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204996.840449:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204996.840453:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204996.840455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204996.840456:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204996.840459:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204996.840460:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.840461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204996.840465:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788204996.840465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788204996.840468:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.840469:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204996.840472:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204996.840473:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204996.840475:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204996.840477:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.840479:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204997.032461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204997.032466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204997.032470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204997.032471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204997.032474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204997.032474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204997.032476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204997.032477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204997.032532:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204997.032536:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204997.032539:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204997.352452:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204997.352458:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204997.352460:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.352463:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204997.352466:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204997.352469:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.352471:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.352473:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.352474:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.800444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204997.800449:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204997.800451:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204997.800455:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204997.800457:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204997.800460:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204997.800461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204997.800463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204997.800465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.864418:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204997.864424:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204997.864426:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.864430:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788204997.864433:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204997.864435:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.864437:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.864439:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.864442:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204997.864454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204997.864458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204997.864460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204997.864462:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788204997.864465:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204997.864466:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204997.864468:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204997.864469:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204997.864471:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204998.056470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204998.056475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204998.056478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204998.056479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204998.056481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204998.056482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204998.056484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204998.056485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204998.056497:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788204998.056499:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788204998.056501:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204998.376462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204998.376467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204998.376469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204998.376473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204998.376476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204998.376479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204998.376480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204998.376482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204998.376484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.824478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204998.824485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204998.824487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204998.824492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204998.824496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204998.824500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204998.824502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204998.824505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204998.824508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204998.888451:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204998.888452:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204998.888458:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204998.888459:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204998.888460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204998.888463:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204998.888466:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788204998.888467:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204998.888469:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204998.888473:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204998.888474:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204998.888477:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204998.888478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204998.888479:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204998.888484:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204998.888485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204998.888488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204998.888491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204999.080449:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204999.080455:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204999.080461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204999.080463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204999.080466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204999.080467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788204999.080470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788204999.080471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204999.080539:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204999.080554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204999.080557:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204999.400471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204999.400476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204999.400478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204999.400481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788204999.400484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204999.400486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204999.400488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204999.400490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204999.400491:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204999.848478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204999.848484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204999.848486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204999.848496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788204999.848500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204999.848504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204999.848507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204999.848509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204999.848512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.912500:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204999.912506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204999.912507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204999.912510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204999.912512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788204999.912516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204999.912516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204999.912521:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.912521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204999.912525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788204999.912544:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788204999.912549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204999.912550:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204999.912553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204999.912555:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204999.912560:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204999.912562:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204999.912565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205000.104516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205000.104548:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205000.104550:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205000.104554:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104556:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205000.104558:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104560:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104562:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205000.104565:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205000.104567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205000.104568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205000.104580:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205000.104582:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205000.104583:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205000.424458:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301127169, at 4301127680 00000100:00000001:3.0:1788205000.424458:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788205000.424460:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301127169, at 4301127680 00000100:00000001:3.0:1788205000.424465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.424469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205000.424471:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000010:0.0:1788205000.424474:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00100000:3.0:1788205000.424474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205000.424479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205000.424484:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.424487:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.424490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.424493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205000.872462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205000.872471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205000.872473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205000.872478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205000.872481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205000.872485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205000.872488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205000.872490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205000.872493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.936491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205000.936498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.936500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205000.936505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205000.936508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205000.936525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.936527:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.936530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.936532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205000.936660:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205000.936666:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205000.936668:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205000.936672:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205000.936675:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205000.936683:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6455 00000400:00000200:0.0:1788205000.936693:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205000.936697:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000400:00000200:0.0:1788205000.936699:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788205000.936702:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788205000.936704:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788205000.936705:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.936710:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:2.0:1788205000.936711:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205000.936714:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788205000.936716:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788205000.936719:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205000.936720:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205000.936723:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205000.936724:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205000.936727:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205000.936730:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788205000.936731:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205000.936734:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205000.936738:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205000.936742:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205000.936744:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205000.936747:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205000.936751:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.936753:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.936756:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.936758:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205000.936761:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205000.936765:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205000.936767:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205000.936771:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205000.936774:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205000.936777:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.936779:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205000.936782:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205000.936790:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205000.936794:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205000.936797:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205000.936799:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.936802:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205000.936805:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205000.936809:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205000.936817:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788205000.936824:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205000.936832:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205000.936842:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205000.936846:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.936851:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.936902:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205000.936906:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205000.936908:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205000.936914:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.936922:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.936923:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.936926:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205000.936929:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205000.936931:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205000.936935:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205918848:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205000.936939:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205000.936943:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205000.936945:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.936947:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205000.936952:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.936954:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205000.936959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205000.936963:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205000.936969:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205000.936975:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918848, portal 25 00000100:00000001:3.0:1788205000.936978:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205000.936980:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205000.936985:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.936992:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205000.936995:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205000.936997:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205000.936999:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205918848, offset 0 00000400:00000200:3.0:1788205000.937006:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205000.937013:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.937019:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205000.937024:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205000.937028:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205000.937031:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205000.937034:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205000.937038:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59034:59034:256:4294967295] 192.168.201.151@tcp LPNI seq info [59014:59014:8:4294967295] 00000400:00000200:3.0:1788205000.937043:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.937052:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205000.937059:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.937063:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205000.937070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205000.937074:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788205000.937080:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205000.937083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788205000.937088:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.937090:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205000.937096:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205000.937100:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205000.937105:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205000.937107:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205000.937110:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205000.937114:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205000.937142:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205000.937145:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205000.937156:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205000.937160:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205000.937162:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205000.937165:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205000.937168:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6455 00000400:00000200:0.0:1788205000.937171:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205000.937174:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205000.937176:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205000.937179:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937181:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.937183:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937185:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205000.937187:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205000.937189:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205000.937190:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205000.937192:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205000.937195:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205000.937197:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205000.937200:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937202:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.937204:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937206:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205000.937209:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205000.937212:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937214:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205000.937216:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205000.937221:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205000.937223:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.937225:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205000.937229:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205000.937232:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.937234:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205000.937238:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205000.937240:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205000.937242:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205000.937244:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.937246:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205000.937247:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205000.937249:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205000.937253:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788205000.937256:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205000.937262:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205000.937269:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205000.937271:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.937275:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937320:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.937325:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937332:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937334:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205000.937340:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205000.937344:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205000.937346:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301132289 00000100:00000010:3.0:1788205000.937351:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4a20. 00000100:00000001:3.0:1788205000.937357:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205000.937362:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301132289 00000100:00000010:3.0:1788205000.937383:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788205000.937387:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205000.937390:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205000.937393:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937395:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205000.937399:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205000/real 1788205000] req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205000.937408:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205000.937409:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205000.937412:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205000.937414:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205000.937416:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205000.937418:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205000.937422:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205000.937431:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937438:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205000.937440:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937442:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.937444:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205000.937447:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937455:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205000.937460:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937466:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205000.937469:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937470:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.937471:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937474:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205000.937476:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205000.937479:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205000.937483:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205000.937485:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205000.937490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205000.937498:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205918848:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205000.937502:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937504:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.937507:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205918848/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937526:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205000.937529:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205000.937531:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.937534:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205000.937536:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205000.937537:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205000.937539:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205000.937542:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788205000.937545:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205000.937547:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205000.937549:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205000.937552:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205000.937553:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937555:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937562:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.937563:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937565:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205000.937566:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205000.937567:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205000.937571:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937578:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.937580:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.937581:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205000.937583:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205000.937586:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205000.937588:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918976:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205000.937591:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205000.937593:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205000.937595:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.937597:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205000.937599:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.937602:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205000.937605:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205000.937608:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205000.937611:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205000.937614:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205918976, portal 10 00000100:00000001:3.0:1788205000.937616:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205000.937617:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205000.937623:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937630:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205000.937632:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205000.937635:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205000.937637:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205918976, offset 0 00000400:00000200:3.0:1788205000.937641:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205000.937644:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.937647:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205000.937650:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205000.937653:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205000.937655:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205000.937658:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205000.937661:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59035:59035:256:4294967295] 192.168.201.151@tcp LPNI seq info [59015:59015:8:4294967295] 00000400:00000200:3.0:1788205000.937664:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.937671:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205000.937675:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.937678:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205000.937682:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205000.937685:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788205000.937690:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205000.937692:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788205000.937696:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.937697:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205000.937700:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205000.937704:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205000.937706:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205000.937708:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205000.937711:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205000.937715:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937723:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.937726:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937733:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937734:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205000.937737:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205000.937741:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205000.937744:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205000.937746:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301128346 00000100:00000001:3.0:1788205000.937749:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.937751:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301128346 00000100:00000001:3.0:1788205000.937753:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.937756:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205000.937758:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937760:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205000.937763:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205000/real 1788205000] req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205000.937771:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205000.937772:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205000.937774:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205000.937776:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205000.937778:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205000.937780:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205000.937783:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205000.937791:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937797:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205000.937798:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937800:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.937801:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205000.937804:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937810:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205000.937814:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937821:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205000.937823:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937825:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.937826:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937828:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205000.937830:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205000.937833:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205000.937836:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205000.937838:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205000.937842:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205000.937849:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205918976:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205000.937853:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937855:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.937858:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205918976/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.937863:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205000.937865:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205000.937867:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.937869:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205000.937870:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205000.937871:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205000.937873:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205000.937875:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788205000.937877:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205000.937878:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205000.937880:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205000.937882:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205000.937884:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937886:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.937889:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.937890:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.937893:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205000.937894:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.937961:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205000.937965:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205000.937968:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205000.937970:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205000.937971:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205000.937975:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205000.937978:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6455 00000400:00000200:0.0:1788205000.937983:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205000.937987:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205000.937990:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205000.937994:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.937996:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.937998:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938000:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205000.938002:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205000.938004:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205000.938005:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205000.938007:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205000.938009:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205000.938011:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205000.938014:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938016:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.938018:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938020:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205000.938022:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205000.938024:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938027:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205000.938028:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205000.938031:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205000.938033:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938035:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205000.938037:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205000.938040:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938042:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205000.938045:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205000.938047:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205000.938050:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205000.938051:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938053:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205000.938054:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205000.938056:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205000.938060:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788205000.938063:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205000.938068:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205000.938076:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205000.938078:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.938082:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205000.938089:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205000.938091:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205000.938094:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205000.938095:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205000.938097:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205000.938099:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205000.938101:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6455 00000400:00000200:0.0:1788205000.938105:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205000.938107:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205000.938110:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205000.938112:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938114:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.938116:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938117:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205000.938119:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205000.938120:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205000.938122:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205000.938124:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205000.938125:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205000.938127:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205000.938130:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938131:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205000.938133:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938135:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205000.938137:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205000.938139:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205000.938141:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205000.938143:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205000.938145:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205000.938147:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938149:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205000.938151:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205000.938153:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938155:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205000.938159:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205000.938161:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205000.938163:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205000.938164:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205000.938166:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205000.938167:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205000.938169:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205000.938172:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788205000.938175:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205000.938180:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205000.938186:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205000.938188:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.938191:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205000.938199:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205000.938203:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6465) 00000100:00000001:3.0:1788205000.938265:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205000.938267:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205000.938269:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205000.938273:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938281:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.938282:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.938284:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205000.938286:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205000.938288:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205000.938291:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205000.938294:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205000.938296:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205000.938297:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.938299:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205000.938302:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.938304:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205000.938308:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205000.938310:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205000.938314:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205000.938317:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919104, portal 4 00000100:00000001:3.0:1788205000.938318:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205000.938320:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205000.938325:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938331:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205000.938334:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205000.938336:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205000.938338:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205919104, offset 0 00000400:00000200:3.0:1788205000.938342:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205000.938346:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.938350:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205000.938353:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205000.938355:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205000.938358:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205000.938360:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205000.938378:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59036:59036:256:4294967295] 192.168.201.151@tcp LPNI seq info [59016:59016:8:4294967295] 00000400:00000200:3.0:1788205000.938383:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.938390:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205000.938395:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.938398:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205000.938402:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205000.938405:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788205000.938410:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205000.938412:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788205000.938416:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.938418:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205000.938420:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205000.938424:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205000.938427:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205000.938429:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205000.938431:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205000.938435:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938443:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.938446:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938452:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.938454:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205000.938456:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205000.938459:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205000.938462:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205000.938465:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301128234 00000100:00000001:3.0:1788205000.938468:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.938470:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301128234 00000100:00000001:3.0:1788205000.938472:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.938474:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205000.938476:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.938478:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205000.938482:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205000/real 1788205000] req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205000.938489:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205000.938491:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205000.938492:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205000.938495:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205000.938496:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205000.938498:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205000.938502:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205000.938521:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938528:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205000.938530:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938532:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.938533:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205000.938537:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938544:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205000.938547:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938554:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205000.938557:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938559:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.938560:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938562:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205000.938564:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205000.938567:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205000.938569:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205000.938571:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205000.938575:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205000.938583:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205000.938587:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205000.938591:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938597:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205000.938599:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.938601:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205000.938603:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205000.938605:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205000.938607:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919232:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205000.938611:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205000.938612:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205000.938614:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205000.938615:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205000.938618:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205000.938620:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205000.938623:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205000.938626:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205000.938629:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205000.938631:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919232, portal 4 00000100:00000001:3.0:1788205000.938634:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205000.938635:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788205000.938640:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938646:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205000.938649:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205000.938651:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205000.938653:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205919232, offset 0 00000400:00000200:3.0:1788205000.938657:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205000.938660:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.938664:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205000.938666:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205000.938669:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205000.938671:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205000.938674:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205000.938677:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59037:59037:256:4294967295] 192.168.201.151@tcp LPNI seq info [59017:59017:8:4294967295] 00000400:00000200:3.0:1788205000.938681:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205000.938687:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205000.938692:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.938695:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205000.938698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205000.938701:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788205000.938705:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205000.938707:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788205000.938711:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205000.938713:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205000.938716:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205000.938719:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205000.938721:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205000.938723:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205000.938725:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205000.938728:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938740:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.938743:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938749:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.938751:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205000.938753:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205000.938756:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205000.938759:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205000.938761:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301128240 00000100:00000001:3.0:1788205000.938763:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.938765:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301128240 00000100:00000001:3.0:1788205000.938767:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301132289 00000100:00000001:3.0:1788205000.938769:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205000.938771:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.938773:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205000.938776:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205000/real 1788205000] req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205000.938783:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205000.938785:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205000.938786:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205000.938788:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205000.938790:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205000.938792:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205000.938795:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205000.938804:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938810:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205000.938812:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938814:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.938815:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205000.938819:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938831:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205000.938835:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938842:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205000.938844:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938846:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205000.938847:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938849:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205000.938851:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205000.938853:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205000.938856:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205000.938858:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205000.938862:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205000.938870:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919232:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205000.938873:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938875:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.938878:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205919104/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938884:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205000.938885:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205000.938887:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205000.938890:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205000.938891:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205000.938893:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205000.938895:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205000.938897:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788205000.938899:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205000.938900:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205000.938902:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205000.938904:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205000.938906:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938907:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205000.938911:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205919232/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205000.938917:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205000.938918:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205000.938919:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205000.938921:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205000.938922:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205000.938924:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205000.938925:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205000.938926:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788205000.938928:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205000.938929:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205000.938930:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205000.938932:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205000.938933:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938934:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205000.938938:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.938938:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.938940:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205000.938941:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205001.128473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205001.128481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205001.128486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205001.128487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205001.128490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205001.128492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205001.128495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205001.128496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205001.128555:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205001.128559:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205001.128562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205001.448489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205001.448499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205001.448515:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205001.448526:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205001.448531:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205001.448536:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.448541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205001.448545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.448550:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.896479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205001.896487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205001.896489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205001.896514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205001.896518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205001.896522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205001.896524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205001.896527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205001.896529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205001.960489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205001.960489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205001.960511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205001.960511:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205001.960516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205001.960516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205001.960522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205001.960523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205001.960528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205001.960529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205001.960533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.960534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205001.960537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205001.960540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205001.960543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205001.960546:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205001.960549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.960552:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205002.152557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205002.152566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205002.152572:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205002.152574:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205002.152577:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205002.152578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205002.152582:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205002.152583:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205002.152611:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205002.152614:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205002.152617:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205002.472514:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205002.472522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205002.472524:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205002.472529:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205002.472533:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205002.472537:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.472539:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205002.472542:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.472544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.920437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205002.920445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205002.920447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.920451:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205002.920454:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205002.920458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.920460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205002.920463:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.920465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205002.984467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205002.984467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205002.984485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205002.984486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205002.984489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.984490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205002.984497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205002.984497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205002.984503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205002.984507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205002.984509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205002.984510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205002.984514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205002.984516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205002.984519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.984520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205002.984524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.984527:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205003.176519:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205003.176522:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205003.176527:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176533:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205003.176537:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176539:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176542:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205003.176546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205003.176550:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205003.176551:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205003.176552:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176553:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205003.176554:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176555:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176556:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205003.176566:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205003.176568:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205003.176569:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176571:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205003.176572:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176574:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205003.176575:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205003.176643:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205003.176648:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205003.176651:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205003.496514:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205003.496524:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205003.496527:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205003.496533:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205003.496537:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205003.496540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205003.496543:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205003.496545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205003.496548:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205003.944488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205003.944495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205003.944497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205003.944502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205003.944505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205003.944508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205003.944510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205003.944513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205003.944515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205004.008498:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205004.008498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205004.008506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205004.008507:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205004.008509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205004.008510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205004.008517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205004.008517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205004.008522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205004.008523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205004.008528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.008528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.008532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205004.008534:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.008536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205004.008537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205004.008540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205004.008542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205004.200564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205004.200569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205004.200572:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205004.200573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205004.200575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205004.200576:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205004.200578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205004.200579:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205004.200593:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205004.200595:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205004.200597:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205004.520435:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205004.520443:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205004.520445:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205004.520459:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205004.520463:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205004.520466:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.520475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205004.520477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.520480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205004.968469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205004.968475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205004.968476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205004.968480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205004.968483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205004.968486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205004.968488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205004.968489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205004.968491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205005.032437:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205005.032456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205005.032458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205005.032463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205005.032467:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205005.032471:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205005.032475:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205005.032477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205005.032477:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205005.032482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205005.032482:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205005.032485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205005.032489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205005.032491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205005.032492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205005.032494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205005.032495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205005.032497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205005.224496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205005.224502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205005.224506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205005.224507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205005.224509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205005.224510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205005.224513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205005.224514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205005.224577:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205005.224583:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205005.224588:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205005.544493:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301132289, at 4301132800 00000100:00000040:1.0:1788205005.544493:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301132289, at 4301132800 00000100:00000001:3.0:1788205005.544494:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205005.544503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205005.544505:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205005.544509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205005.544511:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000010:1.0:1788205005.544511:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788205005.544512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205005.544516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205005.544520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205005.544522:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205005.544524:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205005.992491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205005.992497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205005.992499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205005.992505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205005.992508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205005.992511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205005.992513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205005.992516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205005.992519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205006.056480:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205006.056482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205006.056486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205006.056489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788205006.056494:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205006.056494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205006.056495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205006.056500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056502:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205006.056504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205006.056510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788205006.056511:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205006.056512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205006.056514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205006.056519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205006.056520:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205006.056521:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788205006.056530:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205006.056535:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6460 00000100:00000001:2.0:1788205006.056535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205006.056540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.056541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.056547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.056550:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205006.056552:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205006.056559:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205006.056563:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205006.056568:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056572:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205006.056576:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056580:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205006.056584:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205006.056587:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205006.056589:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205006.056599:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205006.056600:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205006.056602:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205006.056604:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056606:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205006.056607:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056608:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205006.056611:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205006.056613:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056614:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205006.056616:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205006.056618:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205006.056620:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056621:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205006.056623:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205006.056631:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205006.056634:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205006.056637:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205006.056638:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056640:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205006.056642:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205006.056645:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205006.056652:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:0.0:1788205006.056656:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205006.056663:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205006.056670:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205006.056673:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205006.056676:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205006.056689:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205006.056691:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.056695:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205006.056696:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205006.056698:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205006.056701:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205006.056703:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6460 00000400:00000200:0.0:1788205006.056707:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205006.056711:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205006.056713:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205006.056717:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056719:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205006.056721:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788205006.056722:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056727:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205006.056727:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205006.056731:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205006.056732:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205006.056735:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205006.056737:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205006.056739:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205006.056741:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205006.056744:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205006.056745:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205006.056750:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056755:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205006.056759:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205006.056763:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205006.056764:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205006.056767:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788205006.056767:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205006.056772:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056774:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205006.056775:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205006.056775:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205006.056779:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205006.056782:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205006.056782:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.056789:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205919360:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205006.056792:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205006.056793:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056795:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205006.056796:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788205006.056798:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205006.056799:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.056803:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788205006.056804:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056807:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205006.056809:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 00000100:00000010:0.0:1788205006.056810:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 02000000:00000001:3.0:1788205006.056813:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205006.056814:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205006.056818:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205006.056819:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.056821:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788205006.056821:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788205006.056823:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205006.056826:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205006.056829:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000010:0.0:1788205006.056831:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:0.0:1788205006.056836:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205006.056842:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788205006.056845:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788205006.056852:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205006.056854:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205006.056856:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919360, portal 25 00000100:00000040:0.0:1788205006.056860:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.056861:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205006.056864:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000001:0.0:1788205006.056874:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205006.056875:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205006.056879:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.056886:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205006.056889:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205006.056892:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788205006.056893:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205006.056898:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788205006.056899:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205006.056903:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6460 00000400:00000010:3.0:1788205006.056904:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205006.056910:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205919360, offset 0 00000400:00000200:0.0:1788205006.056911:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205006.056919:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205006.056919:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205006.056925:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205006.056928:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205006.056930:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205006.056933:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205006.056937:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788205006.056938:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205006.056943:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205006.056944:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205006.056948:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788205006.056950:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788205006.056953:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788205006.056954:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205006.056956:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788205006.056959:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788205006.056962:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59038:59038:256:4294967295] 192.168.201.151@tcp LPNI seq info [59018:59018:8:4294967295] 00000020:00000040:0.0:1788205006.056964:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788205006.056968:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788205006.056969:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205006.056972:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205006.056975:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056980:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205006.056982:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:0.0:1788205006.056984:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.056988:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788205006.056992:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788205006.056993:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000010:3.0:1788205006.056997:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00080000:0.0:1788205006.056999:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205006.057003:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205006.057004:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788205006.057007:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1788205006.057009:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 02000000:00000001:0.0:1788205006.057012:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205006.057017:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205006.057018:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205006.057020:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:3.1:1788205006.057023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000008:00100000:0.0:1788205006.057025:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205006.057029:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205006.057031:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205006.057032:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205006.057036:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000010:0.0:1788205006.057038:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205006.057041:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205006.057043:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788205006.057044:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:0.0:1788205006.057045:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.057051:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205006.057054:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788205006.057055:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1788205006.057057:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205006.057061:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 02000000:00000010:0.0:1788205006.057063:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000400:00000010:3.0:1788205006.057064:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788205006.057069:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057070:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205006.057075:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205006.057079:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205006.057090:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.057096:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205006.057101:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205006.057104:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057107:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057111:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1788205006.057112:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057117:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.057118:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205006.057121:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205006.057123:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788205006.057125:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205006.057127:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301137409 00000100:00000001:0.0:1788205006.057128:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205006.057130:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205006.057133:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000010:3.0:1788205006.057134:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f44e0. 00000100:00080000:0.0:1788205006.057135:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6460 00000400:00000200:0.0:1788205006.057140:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205006.057142:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1788205006.057143:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205006.057147:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205006.057149:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205006.057150:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301137409 00000100:00000001:0.0:1788205006.057151:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205006.057154:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205006.057156:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205006.057157:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205006.057158:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205006.057158:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205006.057160:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205006.057161:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205006.057164:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000010:3.0:1788205006.057164:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4780. 00000100:00000040:0.0:1788205006.057170:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205006.057170:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205006.057171:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205006.057175:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205006.057176:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205006.057177:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205006.057179:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205006.057181:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788205006.057182:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1788205006.057185:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205006.057189:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.057189:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205006.057191:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205006.057194:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205006.057199:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205006.057202:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:3.0:1788205006.057202:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:0.0:1788205006.057204:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000001:3.0:1788205006.057205:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:0.0:1788205006.057207:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205006.057208:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788205006.057209:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205006.057212:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000010:0.0:1788205006.057214:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000400:00000010:3.0:1788205006.057215:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000020:00000040:0.0:1788205006.057218:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788205006.057219:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1788205006.057223:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205006.057225:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205006.057225:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205006.057229:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205006.057230:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205006.057233:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205006.057237:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000200:3.0:1788205006.057239:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205006.057241:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205006.057249:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205006.057250:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205006.057252:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057256:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057259:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205006.057260:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205006.057262:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.057265:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205006.057268:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057283:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205006.057285:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205006.057289:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6471) 00000100:00000040:3.0:1788205006.057290:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057295:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205006.057297:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057298:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057299:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057301:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205006.057302:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205006.057304:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205006.057307:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205006.057309:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205006.057311:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205006.057316:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205919360:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205006.057318:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057320:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.057321:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205919360/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057325:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205006.057326:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205006.057328:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205006.057330:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205006.057331:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205006.057332:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205006.057333:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205006.057335:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:3.0:1788205006.057337:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205006.057338:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205006.057339:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205006.057340:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205006.057341:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057343:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057348:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205006.057350:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057352:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205006.057354:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205006.057355:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205006.057358:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057386:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.057387:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057390:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205006.057391:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205006.057394:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.057397:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919488:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205006.057399:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205006.057401:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205006.057402:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057403:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205006.057405:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205006.057406:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205006.057409:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205006.057411:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205006.057413:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205006.057415:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919488, portal 10 00000100:00000001:3.0:1788205006.057416:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205006.057417:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205006.057436:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057440:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205006.057442:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205006.057443:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205006.057445:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205919488, offset 0 00000400:00000200:3.0:1788205006.057447:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057450:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057453:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205006.057455:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205006.057456:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205006.057458:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205006.057459:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057461:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59039:59039:256:4294967295] 192.168.201.151@tcp LPNI seq info [59019:59019:8:4294967295] 00000400:00000200:3.0:1788205006.057463:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057468:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205006.057470:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057473:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205006.057475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205006.057477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205006.057480:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205006.057482:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205006.057484:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057485:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205006.057487:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205006.057489:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205006.057491:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205006.057492:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205006.057494:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205006.057496:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057501:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.057503:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057506:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057507:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205006.057509:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205006.057511:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205006.057513:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205006.057514:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301133467 00000100:00000001:3.0:1788205006.057516:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057517:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301133467 00000100:00000001:3.0:1788205006.057519:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057520:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205006.057521:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057522:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205006.057524:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205006.057532:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205006.057533:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205006.057534:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205006.057535:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205006.057536:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205006.057537:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205006.057538:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205006.057543:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057547:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205006.057548:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057549:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057549:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205006.057551:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057555:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205006.057557:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057560:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205006.057561:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057562:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057563:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057564:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205006.057565:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205006.057567:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205006.057569:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205006.057570:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205006.057573:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205006.057577:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919488:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205006.057579:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205006.057581:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057584:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.057585:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057586:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205006.057587:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205006.057588:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.057590:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205006.057591:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205006.057593:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205006.057594:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057595:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205006.057598:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205006.057599:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205006.057602:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205006.057605:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205006.057608:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205006.057610:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919616, portal 4 00000100:00000001:3.0:1788205006.057611:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205006.057613:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205006.057618:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057624:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205006.057626:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205006.057629:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205006.057631:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205919616, offset 0 00000400:00000200:3.0:1788205006.057635:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057638:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057641:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205006.057644:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205006.057646:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205006.057648:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205006.057649:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057651:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59040:59040:256:4294967295] 192.168.201.151@tcp LPNI seq info [59020:59020:8:4294967295] 00000400:00000200:3.0:1788205006.057654:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057658:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205006.057661:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057663:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205006.057665:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205006.057667:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205006.057669:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205006.057671:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205006.057673:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057674:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205006.057676:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205006.057678:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205006.057680:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205006.057681:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205006.057682:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205006.057684:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057689:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.057692:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057698:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057699:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205006.057702:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205006.057704:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205006.057707:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205006.057709:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301133353 00000100:00000001:3.0:1788205006.057712:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057714:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301133353 00000100:00000001:3.0:1788205006.057716:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057718:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205006.057720:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057721:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205006.057725:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205006.057732:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205006.057734:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205006.057735:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205006.057738:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205006.057739:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205006.057741:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205006.057744:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205006.057752:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057758:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205006.057760:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057761:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057762:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205006.057766:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057772:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205006.057775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057779:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205006.057781:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057782:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057783:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057784:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205006.057786:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205006.057787:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205006.057789:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205006.057791:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205006.057793:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205006.057797:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205006.057800:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205006.057802:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057808:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.057809:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057811:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205006.057812:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205006.057814:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.057817:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919744:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205006.057820:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205006.057821:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205006.057823:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205006.057825:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205006.057828:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205006.057830:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205006.057833:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205006.057835:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205006.057838:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205006.057841:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919744, portal 4 00000100:00000001:3.0:1788205006.057842:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205006.057843:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205006.057849:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057855:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205006.057857:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205006.057859:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205006.057861:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205919744, offset 0 00000400:00000200:3.0:1788205006.057865:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057871:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205006.057874:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205006.057876:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205006.057879:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205006.057881:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205006.057884:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59041:59041:256:4294967295] 192.168.201.151@tcp LPNI seq info [59021:59021:8:4294967295] 00000400:00000200:3.0:1788205006.057888:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205006.057894:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205006.057897:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057899:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205006.057902:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205006.057903:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205006.057906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205006.057907:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205006.057910:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205006.057911:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205006.057913:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205006.057915:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205006.057916:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205006.057917:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205006.057919:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205006.057921:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057930:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.057931:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057935:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057936:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205006.057938:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205006.057940:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205006.057942:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205006.057943:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301133359 00000100:00000001:3.0:1788205006.057944:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057946:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301133359 00000100:00000001:3.0:1788205006.057947:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301137409 00000100:00000001:3.0:1788205006.057948:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205006.057949:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.057950:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205006.057952:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205006.057956:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205006.057957:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205006.057958:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205006.057959:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205006.057960:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205006.057962:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205006.057963:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205006.057968:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057971:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205006.057972:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057973:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057974:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205006.057976:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057980:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205006.057982:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.057985:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205006.057987:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057987:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205006.057988:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.057989:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205006.057990:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205006.057992:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205006.057993:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205006.057994:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205006.057996:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205006.058000:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205919744:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205006.058002:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.058003:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.058005:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205919488/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.058009:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205006.058010:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205006.058011:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205006.058013:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205006.058014:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205006.058015:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205006.058017:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205006.058019:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788205006.058021:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205006.058022:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205006.058023:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205006.058026:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205006.058027:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.058028:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.058031:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205919616/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.058038:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205006.058039:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205006.058040:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205006.058042:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205006.058044:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205006.058045:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205006.058047:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205006.058048:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788205006.058050:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205006.058052:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205006.058053:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205006.058055:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205006.058056:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.058058:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205006.058061:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205919744/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205006.058065:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205006.058066:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205006.058067:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205006.058068:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205006.058069:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205006.058070:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205006.058071:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205006.058071:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788205006.058072:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205006.058073:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205006.058074:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205006.058076:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205006.058077:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.058078:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.058080:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205006.058081:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.058082:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205006.058083:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205006.248520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205006.248525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205006.248529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205006.248530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205006.248532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205006.248532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205006.248534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205006.248535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205006.248547:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205006.248549:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205006.248551:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205006.568515:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205006.568522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205006.568524:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205006.568529:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205006.568533:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205006.568536:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.568539:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.568542:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.568545:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.016473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205007.016479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205007.016480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.016484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205007.016486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205007.016489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.016491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.016493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.016495:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.080481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205007.080491:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205007.080493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205007.080493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205007.080500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205007.080501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205007.080504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.080506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205007.080511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205007.080513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205007.080519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205007.080520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205007.080528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205007.080528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205007.080539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205007.080539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205007.080546:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205007.080551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205007.272482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205007.272489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205007.272494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205007.272495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205007.272498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205007.272498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205007.272501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205007.272501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205007.272559:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205007.272564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205007.272567:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205007.592505:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205007.592511:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205007.592513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205007.592517:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205007.592519:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205007.592522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205007.592525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205007.592528:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205007.592530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205008.040471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205008.040478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205008.040480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205008.040486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205008.040489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205008.040492:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205008.040495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205008.040496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205008.040498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205008.104514:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205008.104523:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205008.104526:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205008.104531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205008.104534:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205008.104537:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.104539:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205008.104542:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.104544:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205008.104546:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205008.104551:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205008.104553:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205008.104557:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205008.104559:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205008.104562:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205008.104565:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205008.104567:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205008.104570:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205008.296528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205008.296533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205008.296537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205008.296538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205008.296541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205008.296542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205008.296543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205008.296544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205008.296555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205008.296557:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205008.296559:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205008.616474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205008.616482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205008.616484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205008.616490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205008.616494:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205008.616497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.616500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205008.616502:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.616506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205009.064428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205009.064433:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205009.064435:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205009.064439:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205009.064441:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205009.064444:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205009.064446:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205009.064448:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205009.064449:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205009.128468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205009.128468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205009.128474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205009.128477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205009.128479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205009.128481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205009.128482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205009.128486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205009.128488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205009.128490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205009.128493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205009.128493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205009.128497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205009.128498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205009.128499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205009.128503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205009.128506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205009.128509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205009.320441:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205009.320447:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205009.320450:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205009.320451:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205009.320453:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205009.320454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205009.320457:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205009.320458:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205009.320504:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205009.320507:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205009.320511:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205009.640487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205009.640495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205009.640497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205009.640501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205009.640504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205009.640508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205009.640511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205009.640513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205009.640516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205010.088488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205010.088494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205010.088495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205010.088499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205010.088502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205010.088505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205010.088507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205010.088509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205010.088511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205010.152463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205010.152463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205010.152470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205010.152470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205010.152473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205010.152474:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205010.152480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205010.152481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205010.152485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205010.152486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205010.152490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.152491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205010.152496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205010.152498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205010.152500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205010.152502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205010.152504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.152506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205010.344509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205010.344515:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205010.344517:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205010.344519:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344521:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205010.344523:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344524:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344525:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205010.344528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205010.344530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205010.344531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205010.344541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205010.344542:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205010.344544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205010.664489:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301137409, at 4301137920 00000100:00000040:1.0:1788205010.664489:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301137409, at 4301137920 00000100:00000001:3.0:1788205010.664490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205010.664497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205010.664501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205010.664505:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4780. 00000100:00100000:3.0:1788205010.664505:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205010.664507:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000001:3.0:1788205010.664509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205010.664513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.664516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205010.664519:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.664523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205011.112459:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205011.112465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205011.112467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205011.112471:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205011.112473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205011.112475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205011.112477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205011.112479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205011.112481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.176456:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205011.176464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205011.176466:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205011.176468:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205011.176472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788205011.176478:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205011.176479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205011.176482:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205011.176484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788205011.176488:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205011.176489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205011.176495:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205011.176495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.176500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205011.176504:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6466 00000400:00000200:0.0:1788205011.176515:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205011.176523:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000400:00000200:0.0:1788205011.176524:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205011.176528:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788205011.176533:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788205011.176535:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205011.176538:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.176539:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.176545:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00100000:2.0:1788205011.176548:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205011.176550:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205011.176557:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788205011.176557:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788205011.176564:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:2.0:1788205011.176565:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205011.176569:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788205011.176572:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788205011.176574:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205011.176580:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788205011.176580:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205011.176590:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788205011.176591:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205011.176602:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176605:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.176607:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176609:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205011.176612:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205011.176615:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176620:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205011.176631:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205011.176637:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205011.176641:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.176643:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205011.176646:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205011.176653:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205011.176656:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205011.176660:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205011.176662:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.176665:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205011.176667:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205011.176670:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205011.176678:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788205011.176684:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205011.176693:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205011.176709:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205011.176713:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205011.176718:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205011.176736:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205011.176739:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.176743:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205011.176744:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205011.176746:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205011.176748:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205011.176756:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6466 00000400:00000200:0.0:1788205011.176760:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205011.176762:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205011.176763:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788205011.176764:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205011.176767:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788205011.176768:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205011.176770:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205011.176772:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205011.176777:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205011.176778:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176782:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205011.176787:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205011.176789:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205011.176794:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205011.176795:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788205011.176796:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788205011.176799:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205011.176800:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205011.176804:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788205011.176804:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205011.176808:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788205011.176810:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205011.176813:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.176814:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.176817:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176820:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788205011.176821:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205919872:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788205011.176822:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205011.176827:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176829:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.176829:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205011.176832:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205011.176835:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205011.176836:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205011.176838:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205011.176839:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.176843:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205011.176844:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788205011.176846:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205011.176849:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205011.176850:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 00000100:00000001:0.0:1788205011.176851:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.176855:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205011.176856:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205011.176860:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205011.176862:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205011.176864:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205011.176865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205011.176867:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205011.176869:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788205011.176869:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000001:0.0:1788205011.176870:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205011.176876:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000400:00000010:3.0:1788205011.176877:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788205011.176879:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205011.176883:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205919872, portal 25 00000100:00100000:0.0:1788205011.176884:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205011.176886:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205011.176888:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000001:0.0:1788205011.176893:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205011.176895:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205011.176898:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205011.176903:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205011.176907:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205011.176911:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205011.176914:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788205011.176915:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205011.176919:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:3.0:1788205011.176919:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205011.176922:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205011.176922:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788205011.176927:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205011.176927:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205919872, offset 0 00000100:00080000:0.0:1788205011.176930:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6466 00000400:00000200:0.0:1788205011.176936:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.176936:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205011.176941:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205011.176943:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205011.176946:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205011.176946:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205011.176948:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.176951:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176954:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205011.176954:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:0.0:1788205011.176957:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205011.176959:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205011.176960:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1788205011.176961:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205011.176964:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205011.176966:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205011.176966:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788205011.176970:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205011.176971:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205011.176974:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:0.0:1788205011.176976:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176980:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205011.176980:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59042:59042:256:4294967295] 192.168.201.151@tcp LPNI seq info [59022:59022:8:4294967295] 00000100:00000040:0.0:1788205011.176984:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176986:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205011.176987:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:0.0:1788205011.176990:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205011.176992:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205011.176994:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205011.176997:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205011.177000:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205011.177000:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 02000000:00000001:0.0:1788205011.177005:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205011.177008:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205011.177009:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205011.177012:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000010:3.0:1788205011.177014:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000008:00000001:0.0:1788205011.177015:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.177018:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205011.177021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000010:0.0:1788205011.177022:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205011.177027:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.0:1788205011.177027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 02000000:00000001:0.0:1788205011.177031:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205011.177033:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205011.177034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205011.177035:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205011.177038:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788205011.177038:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 02000000:00000001:0.0:1788205011.177039:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205011.177043:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000800:00000100:3.0:1788205011.177044:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205011.177048:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205011.177048:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00100000:0.0:1788205011.177055:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205011.177055:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205011.177063:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788205011.177066:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205011.177069:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205011.177069:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205011.177072:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000040:0.0:1788205011.177076:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177076:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205011.177082:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205011.177097:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177098:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.177102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205011.177103:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.177111:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205011.177113:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.177114:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205011.177115:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788205011.177117:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205011.177120:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205011.177122:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205011.177123:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6466 00000400:00000200:3.0:1788205011.177131:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205011.177132:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205011.177138:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205011.177138:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301142529 00000100:00080000:0.0:1788205011.177139:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205011.177144:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.177145:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.177146:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:3.0:1788205011.177147:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c60. 00000100:00000001:0.0:1788205011.177148:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205011.177152:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205011.177153:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205011.177155:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205011.177156:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205011.177157:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788205011.177157:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788205011.177160:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205011.177162:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.177163:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205011.177165:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205011.177166:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205011.177170:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205011.177173:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205011.177173:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301142529 00000100:00000001:0.0:1788205011.177175:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205011.177178:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:3.0:1788205011.177179:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4420. 02000000:00000001:0.0:1788205011.177182:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205011.177185:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788205011.177186:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.177190:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205011.177190:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1788205011.177192:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205011.177195:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788205011.177197:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177200:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205011.177201:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205011.177206:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00100000:3.0:1788205011.177208:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205011.177209:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205011.177213:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205011.177214:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205011.177218:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205011.177220:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205011.177222:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205011.177228:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000400:00000001:3.0:1788205011.177228:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788205011.177232:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205011.177232:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205011.177235:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00100000:0.0:1788205011.177239:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205011.177240:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205011.177243:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205011.177248:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205011.177249:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205011.177251:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205011.177255:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205011.177256:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205011.177281:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205011.177284:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6476) 00000100:00000200:3.0:1788205011.177285:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177294:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205011.177296:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177298:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177300:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205011.177304:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177317:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205011.177322:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177328:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205011.177330:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177358:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177360:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177381:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205011.177383:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205011.177387:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205011.177391:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205011.177393:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205011.177398:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205011.177406:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205919872:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205011.177411:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177413:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.177416:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205919872/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177427:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205011.177429:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205011.177431:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205011.177434:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205011.177436:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205011.177438:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205011.177440:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205011.177442:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788205011.177445:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205011.177447:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205011.177448:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205011.177451:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205011.177452:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177454:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177460:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205011.177462:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177464:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205011.177465:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205011.177466:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205011.177470:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177477:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205011.177478:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177480:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205011.177481:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205011.177484:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205011.177486:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920000:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205011.177489:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205011.177491:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205011.177493:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177495:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205011.177498:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205011.177499:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205011.177503:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205011.177506:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205011.177509:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205011.177511:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920000, portal 10 00000100:00000001:3.0:1788205011.177513:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205011.177515:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205011.177521:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177527:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205011.177529:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205011.177531:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205011.177533:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205920000, offset 0 00000400:00000200:3.0:1788205011.177537:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205011.177540:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.177543:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205011.177546:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205011.177549:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205011.177551:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205011.177561:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205011.177565:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59043:59043:256:4294967295] 192.168.201.151@tcp LPNI seq info [59023:59023:8:4294967295] 00000400:00000200:3.0:1788205011.177569:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.177575:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205011.177579:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.177582:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205011.177586:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205011.177589:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205011.177594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205011.177596:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205011.177599:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.177601:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205011.177604:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205011.177607:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205011.177610:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205011.177612:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205011.177615:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205011.177618:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177625:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.177628:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177634:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177636:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205011.177639:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205011.177642:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205011.177646:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205011.177647:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301138587 00000100:00000001:3.0:1788205011.177650:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.177652:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301138587 00000100:00000001:3.0:1788205011.177654:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.177656:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205011.177658:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177660:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205011.177664:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205011.177672:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205011.177673:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205011.177675:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205011.177677:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205011.177679:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205011.177681:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205011.177684:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205011.177693:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177705:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205011.177706:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177708:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177709:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205011.177713:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177720:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205011.177724:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177731:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205011.177733:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177734:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177736:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177737:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205011.177740:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205011.177742:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205011.177745:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205011.177747:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205011.177750:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205011.177757:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920000:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205011.177761:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205011.177764:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177770:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205011.177772:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177775:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205011.177776:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205011.177778:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205011.177781:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920128:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205011.177784:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205011.177786:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205011.177787:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177789:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205011.177792:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205011.177794:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205011.177797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205011.177799:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205011.177802:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205011.177805:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920128, portal 4 00000100:00000001:3.0:1788205011.177807:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205011.177808:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205011.177813:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177818:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205011.177820:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205011.177822:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205011.177825:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205920128, offset 0 00000400:00000200:3.0:1788205011.177828:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205011.177832:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.177836:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205011.177838:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205011.177841:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205011.177843:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205011.177846:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205011.177848:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59044:59044:256:4294967295] 192.168.201.151@tcp LPNI seq info [59024:59024:8:4294967295] 00000400:00000200:3.0:1788205011.177852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.177858:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205011.177863:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.177866:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205011.177869:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205011.177870:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205011.177873:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205011.177886:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205011.177888:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.177889:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205011.177891:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205011.177894:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205011.177896:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205011.177897:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205011.177899:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205011.177901:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177906:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.177908:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177912:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177913:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205011.177915:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205011.177917:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205011.177919:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205011.177920:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301138473 00000100:00000001:3.0:1788205011.177921:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.177923:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301138473 00000100:00000001:3.0:1788205011.177924:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.177925:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205011.177926:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.177927:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205011.177929:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205011.177934:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205011.177935:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205011.177936:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205011.177937:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205011.177938:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205011.177939:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205011.177941:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205011.177946:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177949:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205011.177950:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177950:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177951:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205011.177953:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177956:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205011.177958:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177962:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205011.177963:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177964:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.177965:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.177966:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205011.177968:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205011.177969:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205011.177971:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205011.177972:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205011.177974:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205011.177979:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920128:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205011.177981:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205011.177983:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.177986:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205011.177987:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177988:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205011.177989:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205011.177990:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205011.177992:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920256:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205011.177994:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205011.177995:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205011.177995:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205011.177996:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205011.177998:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205011.177999:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205011.178001:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205011.178002:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205011.178004:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205011.178006:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920256, portal 4 00000100:00000001:3.0:1788205011.178007:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205011.178008:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788205011.178011:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178014:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205011.178016:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205011.178017:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205011.178018:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205920256, offset 0 00000400:00000200:3.0:1788205011.178021:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205011.178023:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.178025:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205011.178027:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205011.178028:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205011.178030:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205011.178031:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205011.178033:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59045:59045:256:4294967295] 192.168.201.151@tcp LPNI seq info [59025:59025:8:4294967295] 00000400:00000200:3.0:1788205011.178036:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205011.178039:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205011.178041:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.178043:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205011.178045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205011.178047:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205011.178049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205011.178051:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205011.178053:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205011.178055:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205011.178056:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205011.178058:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205011.178060:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205011.178061:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205011.178062:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205011.178064:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178068:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.178070:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178074:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.178075:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205011.178077:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205011.178079:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205011.178082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205011.178084:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301138479 00000100:00000001:3.0:1788205011.178086:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.178088:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301138479 00000100:00000001:3.0:1788205011.178090:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301142529 00000100:00000001:3.0:1788205011.178092:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205011.178094:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.178095:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205011.178099:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205011.178106:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205011.178108:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205011.178109:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205011.178111:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205011.178113:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205011.178115:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205011.178118:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205011.178127:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178131:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205011.178132:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178133:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.178134:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205011.178136:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178140:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205011.178142:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178145:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205011.178147:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178148:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205011.178149:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178150:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205011.178151:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205011.178153:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205011.178154:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205011.178155:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205011.178157:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205011.178161:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920256:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205011.178163:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178165:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.178166:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205920000/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178170:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205011.178171:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205011.178172:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205011.178173:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205011.178174:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205011.178175:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205011.178176:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205011.178178:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788205011.178179:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205011.178180:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205011.178181:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205011.178183:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205011.178183:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178184:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.178186:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205920128/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178193:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205011.178194:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205011.178195:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205011.178196:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205011.178197:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205011.178197:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205011.178198:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205011.178199:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:3.0:1788205011.178200:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205011.178201:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205011.178202:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205011.178203:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205011.178203:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178204:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205011.178206:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205920256/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205011.178209:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205011.178209:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205011.178210:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205011.178211:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205011.178212:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205011.178212:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205011.178213:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205011.178214:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788205011.178215:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205011.178215:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205011.178216:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205011.178217:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205011.178218:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178219:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.178221:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205011.178222:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.178223:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205011.178224:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205011.368459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205011.368466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205011.368472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205011.368474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205011.368477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205011.368478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205011.368481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205011.368482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205011.368544:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205011.368546:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205011.368548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205011.688473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205011.688480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205011.688481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205011.688485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205011.688487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205011.688490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.688492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205011.688494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.688496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.136471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205012.136478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205012.136480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.136486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205012.136489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205012.136493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.136496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205012.136498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.136501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.200515:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205012.200519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205012.200523:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205012.200527:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205012.200528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205012.200529:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205012.200540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205012.200543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205012.200546:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205012.200548:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205012.200554:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.200556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205012.200558:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205012.200562:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205012.200565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.200569:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205012.200574:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.200578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205012.392521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205012.392526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205012.392531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205012.392533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205012.392536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205012.392537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205012.392540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205012.392541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205012.392555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205012.392558:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205012.392560:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205012.712418:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205012.712424:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205012.712426:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205012.712430:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205012.712433:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205012.712435:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.712438:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205012.712439:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.712441:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205013.160473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205013.160479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205013.160481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205013.160487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205013.160490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205013.160493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205013.160496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205013.160498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205013.160501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205013.224482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205013.224491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205013.224491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205013.224494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205013.224500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788205013.224501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205013.224506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205013.224506:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205013.224512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205013.224514:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205013.224516:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205013.224522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205013.224524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205013.224526:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205013.224531:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205013.224535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205013.224538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205013.224541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416468:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205013.416476:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205013.416478:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205013.416481:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416485:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205013.416487:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416488:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416490:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205013.416493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205013.416496:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205013.416497:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205013.416498:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416499:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205013.416500:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416501:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416502:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205013.416505:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205013.416505:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205013.416506:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416507:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205013.416508:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416508:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205013.416509:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205013.416556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205013.416559:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205013.416562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205013.736468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205013.736476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205013.736477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205013.736482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205013.736486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205013.736489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205013.736492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205013.736494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205013.736496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205014.184437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205014.184446:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205014.184449:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205014.184455:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205014.184459:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205014.184463:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205014.184467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205014.184470:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205014.184474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205014.248480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205014.248480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205014.248488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205014.248489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205014.248490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205014.248492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205014.248497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205014.248498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205014.248503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205014.248504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205014.248507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205014.248509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205014.248510:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205014.248514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205014.248516:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205014.248521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205014.248524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205014.248527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205014.440521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205014.440530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205014.440537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205014.440538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205014.440541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205014.440542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205014.440546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205014.440547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205014.440566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205014.440569:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205014.440573:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205014.760528:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205014.760538:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205014.760541:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205014.760547:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205014.760551:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205014.760556:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205014.760559:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205014.760562:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205014.760566:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205015.208512:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205015.208522:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205015.208525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205015.208531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205015.208536:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205015.208540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205015.208543:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205015.208545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205015.208549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205015.272427:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205015.272435:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205015.272438:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205015.272442:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205015.272447:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205015.272450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205015.272451:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205015.272459:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205015.272459:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205015.272461:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205015.272466:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205015.272467:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205015.272472:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205015.272475:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205015.272476:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205015.272482:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205015.272485:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205015.272487:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205015.464506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205015.464515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205015.464522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205015.464524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205015.464527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205015.464528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205015.464532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205015.464533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205015.464600:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205015.464603:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205015.464606:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205015.784497:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301142529, at 4301143040 00000100:00000040:1.0:1788205015.784497:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301142529, at 4301143040 00000100:00000010:0.0:1788205015.784522:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000010:1.0:1788205015.784526:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000001:3.0:1788205015.784536:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205015.784548:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205015.784551:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205015.784555:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205015.784560:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205015.784564:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205015.784566:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205015.784569:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205015.784571:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.232482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205016.232489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205016.232490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205016.232496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205016.232500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205016.232504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205016.232507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205016.232509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205016.232512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205016.296509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205016.296519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205016.296522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205016.296529:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205016.296532:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205016.296533:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205016.296536:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788205016.296540:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205016.296543:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788205016.296548:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205016.296549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205016.296551:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296553:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788205016.296556:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205016.296561:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788205016.296561:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205016.296561:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296577:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205016.296578:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205016.296584:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205016.296587:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6471 00000100:00000001:0.0:1788205016.296588:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205016.296592:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205016.296597:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205016.296601:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205016.296609:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205016.296612:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205016.296615:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296619:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.296621:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296624:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205016.296627:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205016.296629:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205016.296631:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205016.296634:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205016.296637:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205016.296639:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205016.296642:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296644:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.296646:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296647:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205016.296650:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205016.296653:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296655:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205016.296658:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205016.296661:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205016.296664:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296666:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205016.296668:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205016.296676:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788205016.296679:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205016.296682:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205016.296684:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296686:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205016.296689:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205016.296692:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205016.296699:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788205016.296704:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205016.296712:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205016.296722:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205016.296725:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205016.296730:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205016.296749:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205016.296752:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205016.296755:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205016.296756:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205016.296757:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205016.296760:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205016.296763:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6471 00000400:00000200:1.0:1788205016.296767:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205016.296771:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205016.296773:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205016.296774:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788205016.296776:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.296780:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205016.296780:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205016.296784:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205016.296784:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205016.296789:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205016.296792:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205016.296794:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000040:3.0:1788205016.296795:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205016.296796:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205016.296799:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205016.296803:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205016.296807:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788205016.296812:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1788205016.296814:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205016.296816:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296818:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.296821:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205016.296822:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205016.296823:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205016.296825:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1788205016.296826:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205016.296828:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205016.296831:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00100000:3.0:1788205016.296834:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205920384:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205016.296835:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.296838:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205016.296842:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205016.296844:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205016.296844:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.296847:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205016.296850:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1788205016.296852:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:1.0:1788205016.296857:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.296857:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296862:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205016.296865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:1.0:1788205016.296867:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788205016.296870:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000008:00000001:1.0:1788205016.296871:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.296875:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205016.296879:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000400:00000010:3.0:1788205016.296879:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000020:00000040:1.0:1788205016.296884:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205016.296886:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205016.296887:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205016.296887:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920384, portal 25 00000100:00000001:1.0:1788205016.296892:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205016.296893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1788205016.296894:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205016.296896:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 02000000:00000001:1.0:1788205016.296897:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205016.296903:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000040:3.0:1788205016.296906:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205016.296908:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.296915:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1788205016.296917:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205016.296920:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205016.296924:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205016.296929:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205920384, offset 0 00000100:00000001:1.0:1788205016.296931:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205016.296934:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205016.296935:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788205016.296942:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205016.296947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205016.296956:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205016.296959:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205016.296959:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:1.0:1788205016.296966:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788205016.296966:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205016.296968:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205016.296970:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205016.296972:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788205016.296974:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205016.296977:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1788205016.296981:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6471 00000400:00000200:3.0:1788205016.296983:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205016.296989:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.296990:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59046:59046:256:4294967295] 192.168.201.151@tcp LPNI seq info [59026:59026:8:4294967295] 00000400:00000200:1.0:1788205016.296997:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205016.297001:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788205016.297002:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205016.297007:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297009:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.297012:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297016:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205016.297021:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788205016.297021:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:1.0:1788205016.297024:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205016.297028:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205016.297030:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000200:3.0:1788205016.297032:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000001:1.0:1788205016.297033:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205016.297037:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:3.0:1788205016.297037:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:1.0:1788205016.297044:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788205016.297047:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205016.297048:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788205016.297052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1788205016.297054:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297058:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.1:1788205016.297059:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:1.0:1788205016.297062:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:3.1:1788205016.297063:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00080000:1.0:1788205016.297069:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000100:3.0:1788205016.297069:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297074:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205016.297074:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 02000000:00000001:1.0:1788205016.297078:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205016.297080:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1788205016.297081:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205016.297086:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1788205016.297087:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.297090:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205016.297090:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205016.297092:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000008:00100000:1.0:1788205016.297094:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:3.0:1788205016.297096:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:1.0:1788205016.297098:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.297102:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205016.297102:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788205016.297107:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788205016.297112:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:3.0:1788205016.297112:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788205016.297117:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:3.0:1788205016.297117:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205016.297120:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.297125:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205016.297128:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205016.297128:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205016.297131:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297131:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205016.297137:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205016.297138:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788205016.297143:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205016.297144:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205016.297148:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301147649 00000100:00100000:1.0:1788205016.297153:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788205016.297155:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4720. 00000100:00000001:3.0:1788205016.297161:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205016.297163:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205016.297166:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297167:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301147649 00000100:00000040:1.0:1788205016.297171:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788205016.297179:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4360. 00000100:00000001:3.0:1788205016.297183:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205016.297184:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297186:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788205016.297188:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205016.297189:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297193:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788205016.297195:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205016.297197:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205016.297198:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205016/real 1788205016] req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205016.297200:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205016.297206:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205016.297211:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6471 00000400:00000001:3.0:1788205016.297211:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205016.297214:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205016.297216:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:1.0:1788205016.297218:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.297219:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205016.297222:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000400:00000200:1.0:1788205016.297224:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205016.297225:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1788205016.297229:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788205016.297231:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205016.297235:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297239:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.297241:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297257:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205016.297260:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205016.297261:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205016.297262:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205016.297264:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205016.297268:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205016.297272:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000200:3.0:1788205016.297276:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205016.297278:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297283:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205016.297287:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205016.297291:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205016.297293:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1788205016.297295:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:3.0:1788205016.297296:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297300:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1788205016.297301:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205016.297302:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205016.297306:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205016.297309:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205016.297310:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205016.297315:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205016.297317:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205016.297319:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.297322:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:3.0:1788205016.297323:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788205016.297325:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205016.297332:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297335:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205016.297337:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297338:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297340:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.297341:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1788205016.297342:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000100:00000001:3.0:1788205016.297343:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:1.0:1788205016.297347:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788205016.297347:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205016.297349:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1788205016.297353:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:3.0:1788205016.297354:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 02000000:00000001:1.0:1788205016.297356:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297357:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788205016.297360:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1788205016.297380:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205016.297387:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205920384:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205016.297390:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297392:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788205016.297394:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1788205016.297395:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205920384/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205016.297398:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205016.297404:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:3.0:1788205016.297410:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1788205016.297411:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.297413:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205016.297416:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205016.297419:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1788205016.297420:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1788205016.297421:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205016.297423:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205016.297427:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205016.297430:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788205016.297434:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205016.297434:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205016.297436:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788205016.297439:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205016.297440:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000040:1.0:1788205016.297446:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297446:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205016.297449:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297454:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205016.297460:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297463:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1788205016.297465:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6481) 00000100:00000001:3.0:1788205016.297467:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297471:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205016.297473:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205016.297474:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205016.297480:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297486:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205016.297488:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.297489:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205016.297491:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205016.297493:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205016.297495:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920512:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205016.297498:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205016.297500:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205016.297501:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.297503:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205016.297506:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205016.297507:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205016.297511:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205016.297514:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205016.297517:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205016.297520:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920512, portal 10 00000100:00000001:3.0:1788205016.297521:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205016.297522:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:3.0:1788205016.297528:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297533:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205016.297535:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205016.297537:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205016.297539:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205920512, offset 0 00000400:00000200:3.0:1788205016.297542:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205016.297546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.297549:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205016.297552:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205016.297554:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205016.297556:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205016.297558:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205016.297560:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59047:59047:256:4294967295] 192.168.201.151@tcp LPNI seq info [59027:59027:8:4294967295] 00000400:00000200:3.0:1788205016.297573:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.297579:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205016.297583:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.297586:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205016.297590:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205016.297592:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205016.297595:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205016.297597:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205016.297601:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.297602:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205016.297605:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205016.297608:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205016.297611:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205016.297612:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205016.297615:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205016.297618:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297625:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.297628:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297633:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297635:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205016.297637:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205016.297641:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205016.297644:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205016.297645:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301143707 00000100:00000001:3.0:1788205016.297648:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.297650:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301143707 00000100:00000001:3.0:1788205016.297652:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.297654:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205016.297655:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297657:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205016.297661:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205016/real 1788205016] req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205016.297668:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205016.297669:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205016.297671:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205016.297674:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205016.297675:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205016.297677:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205016.297680:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205016.297688:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297709:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205016.297710:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297712:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.297713:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205016.297717:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297724:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205016.297727:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297733:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205016.297735:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297737:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.297738:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297740:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205016.297742:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205016.297744:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205016.297747:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205016.297749:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205016.297753:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205016.297760:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920512:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205016.297764:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205016.297767:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297773:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205016.297774:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.297776:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205016.297778:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205016.297780:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205016.297782:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920640:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205016.297785:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205016.297787:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205016.297789:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.297790:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205016.297793:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205016.297795:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205016.297799:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205016.297801:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205016.297804:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205016.297807:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920640, portal 4 00000100:00000001:3.0:1788205016.297808:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205016.297810:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 00000100:00000040:3.0:1788205016.297815:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297821:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205016.297823:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205016.297825:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205016.297828:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205920640, offset 0 00000400:00000200:3.0:1788205016.297831:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205016.297835:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.297838:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205016.297841:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205016.297843:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205016.297845:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205016.297847:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205016.297850:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59048:59048:256:4294967295] 192.168.201.151@tcp LPNI seq info [59028:59028:8:4294967295] 00000400:00000200:3.0:1788205016.297854:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.297859:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205016.297864:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.297867:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205016.297871:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205016.297873:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205016.297877:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205016.297879:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205016.297883:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.297885:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205016.297887:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205016.297890:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205016.297893:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205016.297894:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205016.297896:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205016.297900:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297906:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.297909:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297915:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297917:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205016.297920:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205016.297923:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205016.297926:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205016.297927:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301143593 00000100:00000001:3.0:1788205016.297930:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.297931:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301143593 00000100:00000001:3.0:1788205016.297933:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.297935:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205016.297937:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.297939:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205016.297942:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205016/real 1788205016] req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205016.297949:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205016.297950:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205016.297952:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205016.297954:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205016.297955:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205016.297957:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205016.297961:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205016.297968:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297973:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205016.297975:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.297976:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.297978:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205016.297981:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297987:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205016.297991:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.297997:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205016.297999:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298001:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.298002:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298003:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205016.298005:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205016.298008:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205016.298011:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205016.298013:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205016.298017:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205016.298024:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920640:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205016.298028:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205016.298030:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298036:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205016.298038:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.298040:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205016.298041:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205016.298043:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205016.298045:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920768:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205016.298048:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205016.298050:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205016.298052:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205016.298053:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205016.298056:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205016.298058:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205016.298061:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205016.298063:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205016.298065:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205016.298068:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920768, portal 4 00000100:00000001:3.0:1788205016.298069:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205016.298071:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000040:3.0:1788205016.298076:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298082:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205016.298084:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205016.298086:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205016.298088:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205920768, offset 0 00000400:00000200:3.0:1788205016.298092:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205016.298096:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.298099:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205016.298101:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205016.298103:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205016.298105:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205016.298107:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205016.298109:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59049:59049:256:4294967295] 192.168.201.151@tcp LPNI seq info [59029:59029:8:4294967295] 00000400:00000200:3.0:1788205016.298112:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205016.298117:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205016.298121:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.298124:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205016.298127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205016.298130:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205016.298133:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205016.298135:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205016.298139:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205016.298141:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205016.298143:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205016.298147:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205016.298149:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205016.298151:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205016.298153:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205016.298157:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298163:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.298166:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205041 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298172:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.298173:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205016.298176:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205016.298179:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205016.298182:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205016.298184:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301143599 00000100:00000001:3.0:1788205016.298187:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.298189:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301143599 00000100:00000001:3.0:1788205016.298191:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301147649 00000100:00000001:3.0:1788205016.298193:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205016.298195:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.298196:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205016.298199:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205016/real 1788205016] req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205016.298207:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205016.298208:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205016.298209:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205016.298211:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205016.298213:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205016.298215:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205016.298217:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205016.298225:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298231:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205016.298232:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298234:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.298235:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205016.298238:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298260:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205016.298264:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298272:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205016.298274:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298276:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205016.298277:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298279:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205016.298280:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205016.298283:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205016.298286:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205016.298288:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205016.298291:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205016.298299:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205920768:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205016.298302:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298304:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.298307:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205920512/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298313:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205016.298315:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205016.298317:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205016.298319:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205016.298321:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205016.298322:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205016.298324:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205016.298326:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:3.0:1788205016.298328:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205016.298330:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205016.298331:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:3.0:1788205016.298333:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205016.298334:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298336:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.298339:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205920640/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298350:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205016.298352:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205016.298353:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205016.298355:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205016.298356:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205016.298357:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205016.298359:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205016.298360:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:3.0:1788205016.298362:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205016.298380:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205016.298382:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:3.0:1788205016.298385:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205016.298386:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298388:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205016.298391:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205920768/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205041 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205016.298397:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205016.298398:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205016.298399:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205016.298401:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205016.298402:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205016.298403:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205016.298405:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205016.298406:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:3.0:1788205016.298408:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205016.298409:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205016.298410:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00000001:3.0:1788205016.298411:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205016.298412:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298414:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.298418:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205016.298419:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.298421:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205016.298422:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205016.488483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205016.488487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205016.488491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205016.488492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205016.488494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205016.488495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205016.488496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205016.488497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205016.488509:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205016.488511:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205016.488513:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205016.808467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205016.808475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205016.808476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205016.808481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205016.808484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205016.808486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.808488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205016.808490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.808492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.256495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205017.256501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205017.256503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205017.256507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205017.256509:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205017.256512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.256514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205017.256515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.256517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205017.320497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205017.320505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205017.320507:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205017.320508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205017.320519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788205017.320520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205017.320524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205017.320528:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205017.320532:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205017.320534:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205017.320539:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205017.320541:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205017.320546:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205017.320547:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205017.320553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205017.320554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205017.320560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205017.320563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205017.512511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205017.512518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205017.512524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205017.512526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205017.512529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205017.512531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205017.512534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205017.512536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205017.512607:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205017.512616:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205017.512621:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205017.832487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205017.832494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205017.832496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.832501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205017.832504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205017.832508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.832510:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205017.832513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.832515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205018.280475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205018.280482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205018.280485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205018.280490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205018.280493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205018.280498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205018.280500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205018.280502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205018.280505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205018.344462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205018.344469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205018.344471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205018.344477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205018.344481:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205018.344484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205018.344487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205018.344489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205018.344492:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205018.344493:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205018.344495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205018.344496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205018.344499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205018.344500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205018.344502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.344504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205018.344505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.344507:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205018.536479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205018.536486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205018.536498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205018.536499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205018.536502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205018.536503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205018.536506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205018.536507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205018.536521:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205018.536523:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205018.536525:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205018.856481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205018.856489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205018.856491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205018.856497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205018.856500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205018.856504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.856507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205018.856510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.856512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205019.304479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205019.304485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205019.304487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205019.304492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205019.304496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205019.304499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205019.304502:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205019.304504:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205019.304507:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205019.368481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205019.368489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205019.368490:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205019.368491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.368494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205019.368496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205019.368499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205019.368500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205019.368504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205019.368505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205019.368508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205019.368509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205019.368514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205019.368517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205019.368521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.368522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205019.368525:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205019.368527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205019.560483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205019.560489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205019.560496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205019.560497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205019.560500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205019.560502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205019.560505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205019.560506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205019.560552:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205019.560556:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205019.560558:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205019.880463:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205019.880469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205019.880470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205019.880473:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205019.880476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205019.880478:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205019.880480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205019.880482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205019.880484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205020.328444:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205020.328452:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205020.328454:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205020.328459:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205020.328462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205020.328466:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205020.328468:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205020.328471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205020.328473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205020.392462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205020.392462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205020.392469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205020.392469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205020.392472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205020.392473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205020.392479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205020.392479:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205020.392485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205020.392485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205020.392490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.392490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.392495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205020.392497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.392499:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205020.392501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205020.392503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205020.392505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205020.584519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205020.584528:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205020.584530:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205020.584533:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584535:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205020.584537:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584538:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584540:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205020.584543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205020.584545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205020.584546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205020.584558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205020.584560:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205020.584561:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205020.904468:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301147649, at 4301148160 00000100:00000040:1.0:1788205020.904473:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301147649, at 4301148160 00000100:00000001:3.0:1788205020.904475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205020.904481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205020.904483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205020.904487:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00100000:3.0:1788205020.904487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205020.904489:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00000001:3.0:1788205020.904492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205020.904496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.904498:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205020.904500:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.904502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205021.352478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205021.352483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205021.352485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205021.352489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205021.352492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205021.352495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205021.352496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205021.352498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205021.352500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205021.416466:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205021.416466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205021.416473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205021.416473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205021.416475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.416475:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205021.416481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205021.416481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:2.0:1788205021.416482:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205021.416486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205021.416487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205021.416489:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205021.416492:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.416492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205021.416493:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205021.416496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205021.416498:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205021.416499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205021.416505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416505:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205021.416511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.416513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205021.416514:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6476 00000100:00000001:3.0:1788205021.416517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205021.416526:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205021.416538:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205021.416541:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205021.416544:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416547:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.416550:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416552:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205021.416555:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205021.416557:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205021.416560:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205021.416563:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205021.416565:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205021.416567:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205021.416570:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416572:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.416574:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416575:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205021.416578:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205021.416581:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416583:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205021.416585:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205021.416588:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205021.416590:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416591:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205021.416593:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205021.416600:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205021.416602:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205021.416606:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205021.416607:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416609:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205021.416611:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205021.416613:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205021.416619:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205021.416624:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205021.416631:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205021.416637:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205021.416640:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205021.416643:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205021.416658:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205021.416660:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205021.416663:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205021.416664:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205021.416665:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205021.416666:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205021.416668:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6476 00000400:00000200:2.0:1788205021.416670:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205021.416672:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205021.416674:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205021.416676:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416678:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.416680:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416682:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205021.416682:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205021.416686:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205021.416687:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205021.416688:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205021.416688:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1788205021.416692:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788205021.416693:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788205021.416694:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205021.416697:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205021.416700:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416702:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205021.416703:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205021.416704:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416706:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205021.416710:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205021.416714:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416721:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.416724:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788205021.416725:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205021.416728:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205021.416732:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205021.416734:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205021.416736:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.416738:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205021.416739:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205021.416744:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205021.416748:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205021.416750:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205021.416754:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205920896:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205021.416755:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.416761:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1788205021.416762:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000001:3.0:1788205021.416764:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1788205021.416766:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205021.416767:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205021.416771:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205021.416772:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205021.416775:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416777:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205021.416778:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205021.416780:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205021.416782:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000010:2.0:1788205021.416784:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:3.0:1788205021.416788:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416789:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205021.416797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:2.0:1788205021.416798:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788205021.416804:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:2.0:1788205021.416808:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205021.416812:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205021.416813:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000040:2.0:1788205021.416818:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205021.416821:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205920896, portal 25 00000100:00000001:3.0:1788205021.416826:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205021.416829:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000001:2.0:1788205021.416836:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205021.416841:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205021.416842:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205021.416850:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205021.416853:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205021.416856:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205021.416859:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205021.416861:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788205021.416864:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205021.416867:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00080000:2.0:1788205021.416868:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6476 00000100:00000200:3.0:1788205021.416872:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205920896, offset 0 00000400:00000200:2.0:1788205021.416877:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205021.416883:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205021.416883:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205021.416888:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205021.416891:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416894:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.416897:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205021.416898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205021.416901:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205021.416904:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205021.416906:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205021.416908:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:2.0:1788205021.416909:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205021.416913:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205021.416915:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205021.416917:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205021.416917:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788205021.416922:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416926:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205021.416926:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:2.0:1788205021.416930:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416932:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205021.416934:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788205021.416934:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788205021.416942:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788205021.416942:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.416947:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205021.416948:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205021.416951:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205021.416952:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59050:59050:256:4294967295] 192.168.201.151@tcp LPNI seq info [59030:59030:8:4294967295] 02000000:00000001:2.0:1788205021.416955:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416957:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205021.416960:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788205021.416961:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205021.416964:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416965:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205021.416968:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000400:00000200:3.0:1788205021.416971:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:2.0:1788205021.416972:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205021.416977:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205021.416978:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.416982:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205021.416983:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1788205021.416983:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205021.416986:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205021.416991:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000800:00000010:3.0:1788205021.416992:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:2.0:1788205021.416995:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205021.417000:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205021.417003:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205021.417011:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:3.0:1788205021.417011:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788205021.417013:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205021.417020:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788205021.417023:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205021.417028:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788205021.417033:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205021.417036:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205021.417037:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:3.0:1788205021.417039:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00080000:2.0:1788205021.417040:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205021.417043:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205021.417044:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205021.417045:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205021.417047:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205021.417052:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6476 00000400:00000200:3.0:1788205021.417052:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205021.417057:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417059:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000200:2.0:1788205021.417062:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:3.0:1788205021.417063:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00080000:2.0:1788205021.417064:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205021.417067:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1788205021.417068:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.417071:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.417074:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:3.0:1788205021.417074:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205021.417078:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205021.417083:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205021.417085:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205021.417088:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205021.417090:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205021.417092:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788205021.417093:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205021.417097:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205021.417098:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205021.417102:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205021.417106:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205021.417110:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205021.417112:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.417113:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205021.417116:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205021.417117:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205021.417119:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205021.417120:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205021.417121:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205021.417122:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417122:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205021.417125:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205021.417127:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205021.417129:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417129:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205021.417133:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205021.417136:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000400:00000200:3.0:1788205021.417138:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788205021.417139:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205021.417143:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205021.417144:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417144:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301152769 00000100:00000001:2.0:1788205021.417146:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205021.417148:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205021.417150:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205021.417153:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000010:3.0:1788205021.417153:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f47e0. 00000100:00000001:2.0:1788205021.417172:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205021.417176:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205021.417180:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205021.417181:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205021.417183:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417190:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205021.417191:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205021.417197:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6486) 00000100:00000001:3.0:1788205021.417198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301152769 00000100:00000010:3.0:1788205021.417203:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f43c0. 00000100:00000001:3.0:1788205021.417205:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205021.417210:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205021.417213:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417215:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205021.417221:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205021/real 1788205021] req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205021.417230:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205021.417232:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205021.417234:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205021.417237:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205021.417239:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205021.417241:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205021.417244:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205021.417252:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417259:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205021.417260:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417262:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417264:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205021.417268:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417275:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205021.417279:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417285:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205021.417287:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417289:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417290:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417291:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205021.417292:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205021.417294:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205021.417296:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205021.417299:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205021.417303:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205021.417310:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205920896:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205021.417315:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417316:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.417319:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205920896/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417326:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205021.417328:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205021.417330:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205021.417340:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205021.417342:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205021.417344:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205021.417346:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205021.417347:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:3.0:1788205021.417350:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205021.417351:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205021.417353:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788205021.417354:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205021.417355:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417357:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417361:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205021.417376:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417378:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205021.417379:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205021.417380:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205021.417383:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417387:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.417388:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417389:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205021.417391:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205021.417392:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205021.417393:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921024:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205021.417396:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205021.417397:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205021.417398:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417399:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205021.417401:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205021.417403:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205021.417405:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205021.417407:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205021.417409:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205021.417411:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921024, portal 10 00000100:00000001:3.0:1788205021.417412:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205021.417413:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788205021.417417:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417421:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205021.417423:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205021.417424:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205021.417425:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205921024, offset 0 00000400:00000200:3.0:1788205021.417427:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417430:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417433:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205021.417436:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205021.417438:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205021.417440:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205021.417443:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417446:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59051:59051:256:4294967295] 192.168.201.151@tcp LPNI seq info [59031:59031:8:4294967295] 00000400:00000200:3.0:1788205021.417450:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417456:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205021.417460:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417463:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205021.417467:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205021.417470:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205021.417474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205021.417476:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205021.417480:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417481:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205021.417484:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205021.417487:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205021.417490:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205021.417492:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205021.417494:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205021.417497:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417504:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.417513:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417517:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417518:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205021.417521:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205021.417523:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205021.417525:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205021.417527:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301148827 00000100:00000001:3.0:1788205021.417528:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417530:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301148827 00000100:00000001:3.0:1788205021.417531:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417533:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205021.417534:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417535:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205021.417537:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205021/real 1788205021] req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205021.417542:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205021.417543:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205021.417544:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205021.417545:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205021.417546:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205021.417548:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205021.417549:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205021.417554:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417558:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205021.417558:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417559:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417560:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205021.417562:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417565:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205021.417567:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417571:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205021.417572:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417573:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417574:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417575:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205021.417576:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205021.417578:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205021.417580:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205021.417581:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205021.417583:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205021.417588:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921024:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205021.417589:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205021.417591:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417595:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.417595:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417597:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205021.417598:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205021.417599:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205021.417600:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205021.417602:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205021.417603:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205021.417604:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417605:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205021.417608:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205021.417609:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205021.417611:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205021.417612:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205021.417614:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205021.417616:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921152, portal 4 00000100:00000001:3.0:1788205021.417617:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205021.417618:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788205021.417621:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417628:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205021.417630:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205021.417631:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205021.417632:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205921152, offset 0 00000400:00000200:3.0:1788205021.417634:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417636:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417638:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205021.417640:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205021.417641:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205021.417643:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205021.417644:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417646:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59052:59052:256:4294967295] 192.168.201.151@tcp LPNI seq info [59032:59032:8:4294967295] 00000400:00000200:3.0:1788205021.417649:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417652:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205021.417655:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417657:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205021.417659:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205021.417661:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205021.417664:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205021.417665:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205021.417667:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417668:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205021.417670:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205021.417672:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205021.417674:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205021.417675:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205021.417676:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205021.417678:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417682:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.417684:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417691:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417692:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205021.417693:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205021.417695:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205021.417697:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205021.417698:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301148713 00000100:00000001:3.0:1788205021.417699:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417701:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301148713 00000100:00000001:3.0:1788205021.417702:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417703:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205021.417704:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417705:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205021.417708:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205021/real 1788205021] req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205021.417714:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205021.417715:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205021.417716:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205021.417718:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205021.417720:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205021.417722:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205021.417725:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205021.417733:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417739:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205021.417740:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417742:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417743:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205021.417747:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417753:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205021.417757:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417764:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205021.417766:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417767:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417769:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417770:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205021.417772:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205021.417774:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205021.417777:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205021.417779:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205021.417783:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205021.417790:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205021.417793:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205021.417796:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417800:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.417801:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417803:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205021.417804:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205021.417805:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205021.417807:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921280:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205021.417808:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205021.417810:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205021.417811:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205021.417812:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205021.417814:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205021.417815:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205021.417817:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205021.417819:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205021.417821:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205021.417822:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921280, portal 4 00000100:00000001:3.0:1788205021.417823:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205021.417824:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788205021.417828:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417831:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205021.417833:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205021.417834:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205021.417835:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205921280, offset 0 00000400:00000200:3.0:1788205021.417837:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417839:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417841:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205021.417843:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205021.417845:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205021.417846:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205021.417847:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205021.417849:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59053:59053:256:4294967295] 192.168.201.151@tcp LPNI seq info [59033:59033:8:4294967295] 00000400:00000200:3.0:1788205021.417852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205021.417855:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205021.417858:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417859:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205021.417861:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205021.417863:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205021.417865:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205021.417867:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205021.417869:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205021.417871:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205021.417872:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205021.417874:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205021.417876:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205021.417877:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205021.417878:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205021.417880:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417885:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.417886:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205046 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417890:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417891:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205021.417892:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205021.417894:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205021.417896:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205021.417897:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301148719 00000100:00000001:3.0:1788205021.417898:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417899:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301148719 00000100:00000001:3.0:1788205021.417901:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301152769 00000100:00000001:3.0:1788205021.417902:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205021.417903:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.417904:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205021.417908:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205021/real 1788205021] req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205021.417915:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205021.417916:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205021.417917:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205021.417919:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205021.417921:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205021.417923:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205021.417926:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205021.417934:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417941:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205021.417942:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417944:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417945:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205021.417948:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417952:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205021.417955:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417958:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205021.417960:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417960:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205021.417961:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417962:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205021.417963:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205021.417964:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205021.417966:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205021.417967:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205021.417969:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205021.417973:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921280:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205021.417976:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.417977:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.417981:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205921024/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.417987:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205021.417988:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205021.417990:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205021.417992:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205021.417994:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205021.417995:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205021.417997:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205021.417999:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:3.0:1788205021.418002:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205021.418003:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205021.418004:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788205021.418007:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205021.418008:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.418010:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.418013:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205921152/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.418018:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205021.418019:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205021.418020:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205021.418022:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205021.418022:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205021.418023:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205021.418024:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205021.418025:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:3.0:1788205021.418027:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205021.418027:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205021.418028:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788205021.418029:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205021.418030:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.418031:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205021.418032:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205921280/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205046 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205021.418036:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205021.418037:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205021.418037:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205021.418039:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205021.418039:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205021.418040:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205021.418041:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205021.418042:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788205021.418043:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205021.418044:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205021.418044:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788205021.418045:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205021.418046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.418047:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.418049:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205021.418050:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.418051:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205021.418052:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205021.608481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205021.608488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205021.608495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205021.608497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205021.608500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205021.608501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205021.608504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205021.608506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205021.608587:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205021.608590:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205021.608593:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205021.928449:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205021.928455:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205021.928456:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205021.928460:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205021.928463:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205021.928466:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.928468:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.928470:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205021.928473:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205022.376469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205022.376477:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205022.376479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205022.376484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205022.376488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205022.376491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205022.376494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205022.376496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205022.376499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205022.440437:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205022.440439:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205022.440443:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205022.440445:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205022.440446:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205022.440448:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205022.440450:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205022.440454:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205022.440454:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205022.440456:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205022.440459:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205022.440460:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205022.440462:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205022.440467:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205022.440470:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205022.440473:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205022.440475:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205022.440478:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205022.632505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205022.632511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205022.632517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205022.632518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205022.632521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205022.632522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205022.632525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205022.632526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205022.632541:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205022.632543:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205022.632544:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205022.952472:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205022.952479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205022.952482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205022.952487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205022.952491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205022.952495:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205022.952497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205022.952499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205022.952505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.400469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205023.400483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205023.400485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205023.400490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205023.400494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205023.400498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205023.400500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205023.400503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205023.400505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205023.464477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205023.464479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205023.464488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205023.464488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205023.464492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.464492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205023.464500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205023.464500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205023.464505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205023.464505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205023.464511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.464511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.464515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.464520:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.464523:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205023.464524:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205023.464527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205023.464530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205023.656473:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205023.656476:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205023.656480:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656484:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205023.656487:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656489:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656491:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205023.656497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205023.656501:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205023.656502:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205023.656503:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656505:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205023.656506:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656508:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656509:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205023.656514:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205023.656515:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205023.656516:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656518:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205023.656519:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656520:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205023.656522:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205023.656584:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205023.656589:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205023.656591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205023.976490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205023.976497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205023.976499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205023.976505:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205023.976508:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205023.976512:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.976515:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.976518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.976520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205024.424438:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205024.424445:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205024.424447:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205024.424452:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205024.424455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205024.424459:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205024.424461:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205024.424464:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205024.424466:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205024.488455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205024.488463:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205024.488466:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205024.488471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205024.488475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205024.488479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205024.488481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205024.488484:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205024.488487:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205024.488509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205024.488514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205024.488516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205024.488520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205024.488523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205024.488526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205024.488529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205024.488531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205024.488533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205024.680512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205024.680518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205024.680523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205024.680524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205024.680527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205024.680528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205024.680531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205024.680532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205024.680549:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205024.680551:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205024.680554:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205025.000469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205025.000476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205025.000478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205025.000485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205025.000488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205025.000492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205025.000499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205025.000501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205025.000504:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205025.448471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205025.448476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205025.448478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205025.448481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205025.448483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205025.448487:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205025.448488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205025.448490:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205025.448492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205025.512510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205025.512518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205025.512520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205025.512525:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205025.512528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205025.512531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205025.512534:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205025.512536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205025.512539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205025.512639:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205025.512642:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205025.512644:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205025.512646:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205025.512648:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205025.512650:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205025.512652:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205025.512654:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205025.512656:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205025.704464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205025.704471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205025.704476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205025.704477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205025.704480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205025.704481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205025.704485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205025.704486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205025.704546:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205025.704549:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205025.704551:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205026.024453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205026.024460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205026.024462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205026.024467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205026.024471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205026.024475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205026.024478:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301152769, at 4301153280 00000100:00000001:3.0:1788205026.024479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.024482:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.024488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205026.024491:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301152769, at 4301153280 00000100:00000010:1.0:1788205026.024491:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000010:0.0:1788205026.024501:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000001:1.0:1788205026.472485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205026.472492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205026.472494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205026.472499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205026.472503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205026.472507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205026.472509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205026.472511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205026.472514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205026.536478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788205026.536478:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205026.536488:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205026.536491:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205026.536494:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205026.536498:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205026.536499:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205026.536502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205026.536508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788205026.536508:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6481 00000100:00000001:0.0:1788205026.536515:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205026.536519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205026.536521:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205026.536523:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205026.536528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205026.536529:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205026.536533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205026.536535:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205026.536541:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536545:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536547:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536550:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205026.536553:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205026.536555:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205026.536559:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205026.536562:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205026.536565:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205026.536567:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205026.536570:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536572:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536574:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536576:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205026.536579:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205026.536581:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536584:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205026.536587:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205026.536591:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205026.536594:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536596:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205026.536598:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205026.536606:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205026.536610:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205026.536613:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205026.536615:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536618:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205026.536620:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205026.536624:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205026.536631:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788205026.536637:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205026.536645:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205026.536670:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205026.536674:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205026.536679:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205026.536699:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205026.536703:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205026.536705:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205026.536707:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205026.536708:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205026.536711:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205026.536713:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6481 00000400:00000200:2.0:1788205026.536717:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205026.536719:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205026.536721:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205026.536723:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536725:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536727:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536729:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205026.536731:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205026.536732:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205026.536733:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205026.536735:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205026.536737:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205026.536739:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205026.536741:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536743:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536744:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536746:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205026.536748:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205026.536750:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536752:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205026.536753:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205026.536758:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205026.536760:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536762:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205026.536766:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205026.536768:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536770:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205026.536774:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205026.536776:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205026.536778:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205026.536779:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536781:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205026.536782:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205026.536784:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205026.536788:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788205026.536791:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205026.536796:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205026.536803:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205026.536804:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205026.536808:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205026.536815:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205026.536817:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205026.536819:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205026.536821:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205026.536822:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205026.536824:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205026.536827:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6481 00000400:00000200:2.0:1788205026.536829:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205026.536832:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205026.536834:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205026.536836:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536838:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536840:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536842:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205026.536844:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205026.536845:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205026.536847:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205026.536848:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205026.536850:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205026.536852:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205026.536854:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536856:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536858:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536859:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205026.536861:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205026.536863:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536865:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205026.536866:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205026.536868:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205026.536870:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536871:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205026.536873:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205026.536875:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536877:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205026.536880:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205026.536881:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205026.536883:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205026.536884:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536886:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205026.536887:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205026.536888:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205026.536892:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205026.536894:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205026.536899:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205026.536911:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205026.536913:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205026.536916:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205026.536923:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205026.536926:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205026.536929:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205026.536930:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205026.536932:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205026.536934:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205026.536936:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6481 00000400:00000200:2.0:1788205026.536939:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205026.536942:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205026.536944:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205026.536947:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536949:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536951:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536953:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205026.536954:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205026.536956:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205026.536957:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205026.536959:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205026.536965:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205026.536968:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205026.536971:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536973:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205026.536974:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536976:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205026.536979:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205026.536981:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205026.536983:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205026.536985:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205026.536988:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205026.536990:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536991:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205026.536993:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205026.536996:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.536997:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205026.537002:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205026.537004:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205026.537007:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205026.537008:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.537009:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205026.537010:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205026.537012:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205026.537015:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788205026.537018:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205026.537023:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205026.537029:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205026.537031:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205026.537034:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205026.537040:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205026.537044:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6491) 00000100:00000001:3.0:1788205026.537149:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205026.537153:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205026.537155:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205026.537161:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537168:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.537170:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537173:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205026.537175:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205026.537178:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205026.537181:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205921408:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205026.537184:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205026.537188:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205026.537190:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537193:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537197:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205026.537200:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205026.537205:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205026.537209:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205026.537215:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205026.537222:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921408, portal 25 00000100:00000001:3.0:1788205026.537224:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205026.537226:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:3.0:1788205026.537231:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537239:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205026.537241:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205026.537244:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205026.537247:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205921408, offset 0 00000400:00000200:3.0:1788205026.537253:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537261:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537266:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205026.537271:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205026.537274:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205026.537278:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205026.537280:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537283:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59054:59054:256:4294967295] 192.168.201.151@tcp LPNI seq info [59034:59034:8:4294967295] 00000400:00000200:3.0:1788205026.537287:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537296:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205026.537303:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537308:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205026.537314:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205026.537317:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205026.537322:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205026.537325:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205026.537330:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537332:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205026.537338:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205026.537343:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205026.537348:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205026.537350:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205026.537354:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205026.537357:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537376:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537378:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537383:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537384:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205026.537388:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205026.537391:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205026.537393:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301157890 00000100:00000010:3.0:1788205026.537396:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4cc0. 00000100:00000001:3.0:1788205026.537401:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205026.537405:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301157890 00000100:00000010:3.0:1788205026.537407:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4d20. 00000100:00000001:3.0:1788205026.537409:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205026.537411:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205026.537412:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537414:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205026.537417:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205026/real 1788205026] req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205026.537422:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205026.537423:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205026.537425:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205026.537426:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205026.537427:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205026.537429:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205026.537430:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205026.537436:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537440:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205026.537441:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537443:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537444:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205026.537446:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537449:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205026.537452:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537456:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205026.537457:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537458:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537459:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537460:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205026.537461:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205026.537463:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205026.537465:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205026.537466:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205026.537469:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205026.537473:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205921408:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205026.537476:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205026.537478:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537482:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.537483:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537484:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205026.537485:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205026.537486:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205026.537488:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921536:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205026.537490:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205026.537491:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205026.537492:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537493:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537494:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205026.537495:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205026.537498:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205026.537499:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205026.537501:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205026.537503:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921536, portal 10 00000100:00000001:3.0:1788205026.537504:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205026.537505:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788205026.537509:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537513:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205026.537514:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205026.537515:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205026.537517:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205921536, offset 0 00000400:00000200:3.0:1788205026.537519:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537521:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537523:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205026.537525:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205026.537526:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205026.537528:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205026.537529:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537531:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59055:59055:256:4294967295] 192.168.201.151@tcp LPNI seq info [59035:59035:8:4294967295] 00000400:00000200:3.0:1788205026.537533:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537537:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205026.537540:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537542:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205026.537544:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205026.537546:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205026.537549:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205026.537550:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205026.537552:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537553:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205026.537555:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205026.537557:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205026.537559:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205026.537560:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205026.537561:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205026.537563:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537567:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537569:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537573:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537574:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205026.537575:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205026.537578:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205026.537579:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205026.537581:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301153947 00000100:00000001:3.0:1788205026.537582:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537583:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301153947 00000100:00000001:3.0:1788205026.537584:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537586:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205026.537587:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537587:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205026.537589:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205026/real 1788205026] req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205026.537594:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205026.537595:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205026.537596:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205026.537597:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205026.537598:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205026.537599:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205026.537600:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205026.537605:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537608:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205026.537609:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537610:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537611:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205026.537613:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537616:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205026.537618:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537622:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205026.537623:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537624:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537624:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537625:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205026.537626:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205026.537628:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205026.537629:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205026.537630:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205026.537633:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205026.537637:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921536:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205026.537639:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205026.537641:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537644:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.537645:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537646:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205026.537647:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205026.537648:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205026.537649:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921664:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205026.537651:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205026.537652:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205026.537653:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537653:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537655:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205026.537656:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205026.537658:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205026.537659:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205026.537661:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205026.537662:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921664, portal 4 00000100:00000001:3.0:1788205026.537663:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205026.537664:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788205026.537667:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537671:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205026.537672:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205026.537673:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205026.537674:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205921664, offset 0 00000400:00000200:3.0:1788205026.537676:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537678:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537680:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205026.537682:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205026.537683:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205026.537684:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205026.537686:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537687:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59056:59056:256:4294967295] 192.168.201.151@tcp LPNI seq info [59036:59036:8:4294967295] 00000400:00000200:3.0:1788205026.537690:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537693:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205026.537695:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537696:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205026.537698:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205026.537700:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205026.537702:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205026.537714:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205026.537717:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537718:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205026.537720:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205026.537722:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205026.537724:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205026.537725:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205026.537726:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205026.537728:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537732:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537734:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537738:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537739:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205026.537740:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205026.537742:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205026.537744:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205026.537745:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301153833 00000100:00000001:3.0:1788205026.537747:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537748:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301153833 00000100:00000001:3.0:1788205026.537749:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537750:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205026.537751:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537752:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205026.537754:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205026/real 1788205026] req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205026.537758:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205026.537759:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205026.537760:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205026.537761:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205026.537762:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205026.537763:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205026.537765:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205026.537769:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537772:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205026.537773:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537774:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537775:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205026.537777:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537780:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205026.537782:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537785:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205026.537787:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537788:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537789:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537790:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205026.537791:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205026.537793:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205026.537794:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205026.537795:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205026.537797:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205026.537801:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921664:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205026.537803:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205026.537804:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537808:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.537808:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537809:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205026.537810:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205026.537811:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205026.537813:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921792:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205026.537815:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205026.537816:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205026.537817:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205026.537818:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537820:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788205026.537821:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205026.537822:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205026.537823:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205026.537825:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205026.537826:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921792, portal 4 00000100:00000001:3.0:1788205026.537828:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205026.537828:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788205026.537831:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537835:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205026.537836:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205026.537837:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205026.537838:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205921792, offset 0 00000400:00000200:3.0:1788205026.537840:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537842:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537844:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205026.537846:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205026.537847:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205026.537848:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205026.537850:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205026.537852:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59057:59057:256:4294967295] 192.168.201.151@tcp LPNI seq info [59037:59037:8:4294967295] 00000400:00000200:3.0:1788205026.537854:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205026.537857:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205026.537859:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537861:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205026.537862:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205026.537864:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205026.537866:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205026.537868:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205026.537870:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205026.537871:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205026.537872:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205026.537874:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205026.537876:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205026.537876:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205026.537878:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205026.537879:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537887:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537889:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537893:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537893:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205026.537895:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205026.537896:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205026.537898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205026.537899:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301153839 00000100:00000001:3.0:1788205026.537900:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537901:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301153839 00000100:00000001:3.0:1788205026.537902:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301157890 00000100:00000001:3.0:1788205026.537904:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205026.537905:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.537906:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205026.537908:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205026/real 1788205026] req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205026.537911:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205026.537912:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205026.537913:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205026.537914:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205026.537915:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205026.537916:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205026.537917:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205026.537922:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537925:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205026.537926:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537927:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537928:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205026.537930:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537933:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205026.537935:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537938:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205026.537940:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537940:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205026.537941:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537942:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205026.537943:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205026.537944:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205026.537946:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205026.537947:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205026.537949:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205026.537952:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205921792:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205026.537954:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537955:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537957:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205921408/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537961:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205026.537963:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537965:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205026.537966:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205026.537968:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205026.537969:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205026.537970:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205026.537972:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:3.0:1788205026.537974:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205026.537976:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205026.537977:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788205026.537978:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205026.537979:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537980:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537982:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205921536/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.537985:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205026.537986:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205026.537987:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205026.537988:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205026.537988:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205026.537989:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205026.537990:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205026.537991:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:3.0:1788205026.537992:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205026.537993:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205026.537993:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788205026.537995:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205026.537995:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.537996:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.537997:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205921664/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.538001:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205026.538001:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205026.538002:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205026.538004:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205026.538004:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205026.538005:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205026.538006:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205026.538007:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788205026.538008:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205026.538009:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205026.538010:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788205026.538011:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205026.538012:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.538013:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205026.538014:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205921792/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205026.538018:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205026.538018:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205026.538019:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23000. 02000000:00000001:3.0:1788205026.538020:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205026.538021:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205026.538021:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205026.538022:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205026.538023:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788205026.538024:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205026.538024:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205026.538025:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788205026.538026:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205026.538026:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.538028:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.538032:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205026.538033:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.538034:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205026.538035:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205026.538148:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205026.538150:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205026.538151:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205026.538153:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205026.538154:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205026.538156:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205026.538158:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205026.538158:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205026.538160:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205026.728495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205026.728501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205026.728507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205026.728508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205026.728511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205026.728512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205026.728515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205026.728516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205026.728533:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205026.728535:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205026.728538:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205027.048476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205027.048483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205027.048485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205027.048490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205027.048500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205027.048504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205027.048507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205027.048509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205027.048511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205027.496483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205027.496490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205027.496491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205027.496496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205027.496500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205027.496504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205027.496506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205027.496509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205027.496512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205027.560468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205027.560468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205027.560476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205027.560477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205027.560479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205027.560480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205027.560486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205027.560487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205027.560493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205027.560494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205027.560497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205027.560499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205027.560502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205027.560506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205027.560509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205027.560511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205027.560515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205027.560518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205027.752477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205027.752484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205027.752490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205027.752492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205027.752494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205027.752496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205027.752499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205027.752500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205027.752555:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205027.752558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205027.752561:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205028.072480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205028.072486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205028.072487:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205028.072491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205028.072494:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205028.072497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205028.072499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205028.072501:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205028.072503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.520481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205028.520489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205028.520491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205028.520497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205028.520501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205028.520504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.520507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.520510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.520512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205028.584480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205028.584487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205028.584489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205028.584495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205028.584499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205028.584502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205028.584505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205028.584508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205028.584511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.584722:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205028.584725:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205028.584726:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205028.584729:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205028.584731:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205028.584733:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205028.584736:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205028.584737:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205028.584739:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205028.776508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205028.776515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205028.776520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205028.776522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205028.776525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205028.776526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205028.776529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205028.776530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205028.776544:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205028.776546:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205028.776548:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205029.096572:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205029.096577:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205029.096578:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205029.096582:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205029.096585:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205029.096587:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205029.096589:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205029.096591:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205029.096592:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205029.544473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205029.544480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205029.544482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205029.544487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205029.544490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205029.544494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205029.544496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205029.544498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205029.544501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205029.608486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205029.608486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205029.608493:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205029.608494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205029.608498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205029.608498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205029.608505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205029.608506:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205029.608511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205029.608512:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205029.608515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205029.608516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205029.608521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205029.608525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205029.608529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205029.608529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205029.608533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205029.608536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205029.800478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205029.800485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205029.800490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205029.800492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205029.800495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205029.800497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205029.800500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205029.800501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205029.800564:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205029.800568:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205029.800571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205030.120476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205030.120483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205030.120486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205030.120492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205030.120496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205030.120499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.120504:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205030.120506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.120509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205030.568467:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205030.568474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205030.568476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205030.568482:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205030.568485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205030.568489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205030.568491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205030.568494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205030.568497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205030.632461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205030.632461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205030.632467:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205030.632468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205030.632469:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205030.632471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205030.632475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205030.632476:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205030.632479:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205030.632480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205030.632482:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205030.632484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.632485:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205030.632489:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205030.632492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205030.632495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.632495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205030.632499:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205030.824503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205030.824512:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205030.824514:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205030.824517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824520:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205030.824522:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824524:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824526:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205030.824530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205030.824533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205030.824534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205030.824551:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205030.824553:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205030.824556:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205031.144481:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301157890, at 4301158400 00000100:00000001:3.0:1788205031.144484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205031.144491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205031.144495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205031.144496:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00100000:3.0:1788205031.144499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205031.144503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205031.144507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205031.144509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205031.144512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205031.144515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205031.144586:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301157890, at 4301158400 00000100:00000010:0.0:1788205031.144593:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000001:1.0:1788205031.592447:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205031.592454:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205031.592457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205031.592462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205031.592465:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205031.592468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205031.592469:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205031.592471:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205031.592473:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.656478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205031.656479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788205031.656485:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205031.656486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205031.656487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205031.656491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205031.656492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205031.656496:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1788205031.656498:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205031.656499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205031.656500:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205031.656505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205031.656505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205031.656506:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205031.656510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205031.656510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205031.656512:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205031.656517:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205031.656521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205031.656523:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6486 00000100:00000001:0.0:1788205031.656525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205031.656525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.656529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.656533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205031.656538:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205031.656558:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205031.656562:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205031.656566:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656570:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.656572:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656574:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205031.656578:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205031.656580:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205031.656582:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205031.656585:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205031.656587:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205031.656589:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205031.656592:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656594:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.656596:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656598:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205031.656601:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205031.656604:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656606:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205031.656609:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205031.656612:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205031.656615:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.656617:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205031.656619:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205031.656628:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205031.656632:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205031.656635:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205031.656637:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.656640:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205031.656642:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205031.656646:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205031.656652:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205031.656659:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205031.656669:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205031.656679:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205031.656682:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205031.656687:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205031.656714:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205031.656717:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205031.656720:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205031.656722:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205031.656723:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205031.656726:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205031.656730:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6486 00000400:00000200:2.0:1788205031.656734:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205031.656738:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205031.656740:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205031.656743:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788205031.656744:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205031.656748:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205031.656749:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205031.656753:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205031.656753:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656757:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:0.0:1788205031.656761:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205031.656761:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205031.656763:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205031.656766:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205031.656769:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205031.656773:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205031.656776:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788205031.656777:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205031.656780:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205031.656784:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205031.656786:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205031.656788:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205031.656790:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788205031.656793:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205031.656795:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.656797:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:0.0:1788205031.656802:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205921920:192.168.201.151@tcp:250:kworker.0 00000020:00000040:2.0:1788205031.656803:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:0.0:1788205031.656810:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788205031.656810:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656815:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.656816:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205031.656817:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205031.656820:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205031.656824:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205031.656825:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205031.656830:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205031.656832:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.656839:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205031.656842:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205031.656845:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000100:00000001:2.0:1788205031.656845:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205031.656852:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1788205031.656854:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000008:00000001:2.0:1788205031.656856:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.656860:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205031.656863:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000100:00000200:0.0:1788205031.656864:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205921920, portal 25 00000020:00000040:2.0:1788205031.656868:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:0.0:1788205031.656869:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205031.656871:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 02000000:00000001:2.0:1788205031.656873:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205031.656876:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.656880:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1788205031.656881:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205031.656882:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205031.656886:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205031.656893:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:0.0:1788205031.656894:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205031.656899:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656899:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205031.656902:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205031.656905:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205921920, offset 0 00000100:00100000:2.0:1788205031.656908:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205031.656914:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656921:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1788205031.656922:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205031.656926:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.656929:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:2.0:1788205031.656934:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205031.656935:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205031.656941:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205031.656946:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205031.656952:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656952:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.656958:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59058:59058:256:4294967295] 192.168.201.151@tcp LPNI seq info [59038:59038:8:4294967295] 00000400:00000200:0.0:1788205031.656963:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788205031.656969:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1788205031.656975:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788205031.656976:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205031.656990:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205031.656993:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.0:1788205031.656997:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205031.656998:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205031.657004:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6486 00000800:00000010:0.0:1788205031.657005:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000400:00000200:2.0:1788205031.657010:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205031.657016:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000200:0.0:1788205031.657017:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:2.0:1788205031.657019:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1788205031.657023:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1788205031.657025:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.657030:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.657034:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:0.1:1788205031.657037:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205031.657038:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205031.657043:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:0.1:1788205031.657044:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000020:00000040:2.0:1788205031.657045:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205031.657049:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205031.657052:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000100:0.0:1788205031.657053:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788205031.657057:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000010:0.0:1788205031.657058:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000020:00000001:2.0:1788205031.657063:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205031.657068:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657069:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205031.657071:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.657074:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657078:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205031.657083:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205031.657085:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:2.0:1788205031.657091:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205031.657093:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1788205031.657097:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000200:0.0:1788205031.657098:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205031.657104:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205031.657110:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657111:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788205031.657114:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1788205031.657116:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205031.657119:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205031.657124:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205031.657125:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657126:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205031.657130:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205031.657134:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205031.657134:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205031.657139:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.657143:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205031.657144:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657148:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301163009 00000100:00000010:2.0:1788205031.657150:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205031.657154:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000010:0.0:1788205031.657155:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 02000000:00000001:2.0:1788205031.657156:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205031.657160:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.657161:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205031.657162:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788205031.657165:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788205031.657165:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205031.657171:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:0.0:1788205031.657173:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301163009 00000100:00000001:2.0:1788205031.657175:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205031.657179:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff90c0. 00000100:00100000:2.0:1788205031.657183:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205031.657185:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205031.657191:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205031.657196:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205031.657197:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.657201:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657202:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788205031.657208:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205031.657212:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205031/real 1788205031] req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205031.657230:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788205031.657233:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788205031.657234:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1788205031.657236:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205031.657239:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000100:00080000:2.0:1788205031.657241:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:0.0:1788205031.657244:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000100:00000001:2.0:1788205031.657244:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1788205031.657248:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00080000:2.0:1788205031.657249:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205031.657253:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788205031.657254:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205031.657259:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6486 00000100:00000200:0.0:1788205031.657261:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205031.657266:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205031.657274:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205031.657277:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1788205031.657280:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205031.657283:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.657289:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.657293:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205031.657295:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205031.657297:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205031.657298:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657301:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657303:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205031.657303:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205031.657307:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00100000:0.0:1788205031.657310:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205031.657311:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205031.657315:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:0.0:1788205031.657317:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205031.657320:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205031.657323:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205031.657326:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:0.0:1788205031.657333:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788205031.657335:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205031.657336:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657339:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657341:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205031.657341:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205031.657346:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205031.657349:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205031.657351:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:0.0:1788205031.657353:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788205031.657355:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205031.657357:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205031.657360:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205031.657382:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205031.657387:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205031.657390:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205031.657396:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205031.657396:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205031.657401:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205031.657406:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.657409:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1788205031.657410:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205921920:192.168.201.151@tcp:250:kworker.0 00000008:00100000:2.0:1788205031.657412:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:0.0:1788205031.657418:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788205031.657418:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.657420:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657422:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1788205031.657426:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000040:0.0:1788205031.657428:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205921920/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205031.657430:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205031.657435:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205031.657437:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205031.657441:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205031.657444:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205031.657446:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205031.657450:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:0.0:1788205031.657454:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788205031.657454:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.657459:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1788205031.657461:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1788205031.657462:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205031.657468:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205031.657471:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205031.657475:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000100:00000001:2.0:1788205031.657475:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1788205031.657479:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1788205031.657480:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205031.657484:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 00000100:00000040:2.0:1788205031.657488:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205031.657490:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205031.657493:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205031.657497:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:2.0:1788205031.657501:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657503:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205031.657506:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205031.657508:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6496) 00000100:00000001:0.0:1788205031.657509:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657517:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205031.657518:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657520:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205031.657521:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205031.657522:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205031.657526:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657532:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205031.657532:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.657542:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205031.657544:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205031.657546:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205031.657549:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922048:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205031.657552:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205031.657554:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205031.657555:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.657557:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205031.657560:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205031.657561:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.657565:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205031.657567:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205031.657571:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205031.657574:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922048, portal 10 00000100:00000001:0.0:1788205031.657575:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205031.657577:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205031.657583:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657589:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205031.657592:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205031.657594:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205031.657596:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205922048, offset 0 00000400:00000200:0.0:1788205031.657600:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657604:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.657607:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205031.657610:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205031.657612:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205031.657615:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205031.657617:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657620:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59059:59059:256:4294967295] 192.168.201.151@tcp LPNI seq info [59039:59039:8:4294967295] 00000400:00000200:0.0:1788205031.657625:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.657631:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205031.657635:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.657639:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205031.657643:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205031.657646:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205031.657650:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205031.657652:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205031.657656:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.657657:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205031.657660:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205031.657664:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205031.657667:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205031.657668:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205031.657671:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205031.657674:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657682:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.657685:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657692:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657693:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205031.657697:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205031.657700:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205031.657703:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205031.657706:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301159067 00000100:00000001:0.0:1788205031.657708:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.657710:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301159067 00000100:00000001:0.0:1788205031.657712:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.657714:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205031.657716:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657718:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205031.657721:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205031/real 1788205031] req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205031.657728:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205031.657730:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205031.657731:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205031.657734:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205031.657735:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205031.657737:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205031.657740:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205031.657748:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657755:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205031.657756:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657757:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657759:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205031.657762:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657766:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205031.657769:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657773:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205031.657774:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657775:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657776:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657777:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205031.657778:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205031.657780:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205031.657782:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205031.657783:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205031.657786:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205031.657790:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922048:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205031.657792:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205031.657794:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657798:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205031.657798:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.657800:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205031.657801:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205031.657802:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205031.657804:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922176:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205031.657806:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205031.657807:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205031.657809:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.657811:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205031.657814:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205031.657816:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.657819:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205031.657822:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205031.657825:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205031.657827:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922176, portal 4 00000100:00000001:0.0:1788205031.657829:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205031.657830:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:0.0:1788205031.657836:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657842:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205031.657845:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205031.657847:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205031.657850:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205922176, offset 0 00000400:00000200:0.0:1788205031.657853:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657857:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.657860:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205031.657862:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205031.657864:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205031.657866:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205031.657867:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205031.657869:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59060:59060:256:4294967295] 192.168.201.151@tcp LPNI seq info [59040:59040:8:4294967295] 00000400:00000200:0.0:1788205031.657871:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.657875:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205031.657878:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.657880:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205031.657883:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205031.657884:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205031.657887:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205031.657889:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205031.657891:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.657892:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205031.657895:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205031.657896:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205031.657898:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205031.657899:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205031.657901:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205031.657903:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657908:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.657910:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657914:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657915:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205031.657917:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205031.657919:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205031.657921:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205031.657922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301158953 00000100:00000001:0.0:1788205031.657923:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.657925:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301158953 00000100:00000001:0.0:1788205031.657926:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.657927:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205031.657928:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.657929:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205031.657931:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205031/real 1788205031] req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205031.657935:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205031.657936:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205031.657937:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205031.657938:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205031.657939:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205031.657940:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205031.657942:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205031.657946:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657949:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205031.657950:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657951:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657952:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205031.657954:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657957:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205031.657959:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.657964:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205031.657965:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657966:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.657967:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.657968:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205031.657970:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205031.657971:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205031.657973:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205031.657975:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205031.657989:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205031.657996:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922176:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205031.658000:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205031.658003:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658009:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205031.658010:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.658012:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205031.658014:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205031.658016:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205031.658019:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922304:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205031.658022:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205031.658024:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205031.658025:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205031.658026:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205031.658030:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205031.658032:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205031.658034:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205031.658036:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205031.658038:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205031.658040:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922304, portal 4 00000100:00000001:0.0:1788205031.658041:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205031.658042:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205031.658046:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658050:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205031.658051:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205031.658052:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205031.658054:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205922304, offset 0 00000400:00000200:0.0:1788205031.658056:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205031.658058:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.658060:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205031.658062:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205031.658064:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205031.658065:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205031.658066:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205031.658068:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59061:59061:256:4294967295] 192.168.201.151@tcp LPNI seq info [59041:59041:8:4294967295] 00000400:00000200:0.0:1788205031.658070:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205031.658074:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205031.658077:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.658078:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205031.658081:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205031.658082:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205031.658085:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205031.658086:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205031.658089:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205031.658090:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205031.658092:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205031.658094:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205031.658095:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205031.658096:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205031.658098:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205031.658100:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658105:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.658106:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658110:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.658111:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205031.658113:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205031.658114:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205031.658116:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205031.658117:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301158959 00000100:00000001:0.0:1788205031.658118:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.658120:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301158959 00000100:00000001:0.0:1788205031.658121:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301163009 00000100:00000001:0.0:1788205031.658122:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205031.658123:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.658124:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205031.658126:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205031/real 1788205031] req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205031.658130:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205031.658130:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205031.658131:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205031.658132:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205031.658133:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205031.658134:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205031.658136:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205031.658141:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658150:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205031.658151:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658153:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.658154:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205031.658157:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658163:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205031.658167:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658173:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205031.658175:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658177:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205031.658178:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658179:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205031.658181:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205031.658183:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205031.658186:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205031.658189:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205031.658193:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205031.658201:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922304:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205031.658204:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658206:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.658209:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205922048/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658215:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205031.658217:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205031.658219:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205031.658222:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205031.658223:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205031.658225:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205031.658226:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205031.658228:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:0.0:1788205031.658230:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205031.658230:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205031.658231:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205031.658233:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205031.658234:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658235:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.658237:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205922176/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658242:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205031.658243:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205031.658244:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205031.658246:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205031.658247:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205031.658248:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205031.658250:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205031.658251:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:0.0:1788205031.658253:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205031.658254:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205031.658255:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205031.658257:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205031.658258:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658259:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205031.658262:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205922304/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205031.658268:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205031.658269:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205031.658271:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205031.658273:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205031.658274:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205031.658275:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205031.658278:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205031.658279:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205031.658281:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205031.658282:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205031.658283:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205031.658285:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205031.658286:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658288:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205031.658291:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205031.658292:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205031.658294:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205031.658296:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205031.848564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205031.848581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205031.848588:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205031.848589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205031.848593:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205031.848594:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205031.848597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205031.848599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205031.848664:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205031.848669:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205031.848673:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205032.168507:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205032.168515:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205032.168518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205032.168523:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205032.168526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205032.168530:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205032.168532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205032.168535:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205032.168538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205032.616507:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205032.616516:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205032.616519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205032.616525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205032.616530:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205032.616536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205032.616539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205032.616542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205032.616546:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205032.680436:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205032.680445:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205032.680448:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205032.680450:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205032.680452:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205032.680454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205032.680460:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205032.680466:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205032.680466:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205032.680471:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205032.680472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205032.680475:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205032.680478:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205032.680482:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205032.680485:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205032.680486:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205032.680489:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205032.680494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205032.872554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205032.872563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205032.872570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205032.872571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205032.872575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205032.872577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205032.872580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205032.872581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205032.872602:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205032.872605:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205032.872608:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205033.192487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205033.192495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205033.192497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.192502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205033.192505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205033.192509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.192512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205033.192514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.192516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.640478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205033.640485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205033.640487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.640492:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205033.640495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205033.640499:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.640501:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.640503:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.640506:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.704512:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205033.704520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205033.704522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.704526:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205033.704530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205033.704534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.704548:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205033.704550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.704553:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205033.704655:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205033.704658:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205033.704659:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205033.704662:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205033.704664:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205033.704666:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205033.704668:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205033.704670:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205033.704672:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205033.896480:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205033.896483:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205033.896486:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896491:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205033.896493:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896495:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896497:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205033.896503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205033.896507:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205033.896508:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205033.896509:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896511:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205033.896512:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896514:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896515:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205033.896520:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205033.896521:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205033.896522:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896524:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205033.896525:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896527:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205033.896528:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205033.896596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205033.896600:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205033.896603:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205034.216497:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205034.216503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205034.216506:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205034.216511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205034.216514:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205034.216518:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205034.216520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205034.216523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205034.216525:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205034.664570:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205034.664578:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205034.664580:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205034.664585:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205034.664588:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205034.664592:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205034.664594:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205034.664596:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205034.664599:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205034.728454:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205034.728455:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205034.728465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205034.728467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205034.728471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205034.728475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205034.728475:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205034.728481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205034.728482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205034.728485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205034.728488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205034.728489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205034.728494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205034.728498:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205034.728502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205034.728502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205034.728506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205034.728511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205034.920568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205034.920573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205034.920578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205034.920578:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205034.920580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205034.920581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205034.920583:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205034.920584:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205034.920604:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205034.920605:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205034.920607:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205035.240504:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205035.240515:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205035.240518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205035.240525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205035.240530:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205035.240535:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.240538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205035.240541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.240544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205035.688488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205035.688500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205035.688502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205035.688507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205035.688510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205035.688513:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205035.688515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205035.688517:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205035.688519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205035.752492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205035.752501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205035.752501:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205035.752503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205035.752508:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788205035.752509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205035.752513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205035.752513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205035.752518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205035.752520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205035.752522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205035.752525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.752527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205035.752529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205035.752532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.752539:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205035.752542:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.752545:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205035.944491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205035.944500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205035.944508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205035.944509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205035.944513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205035.944514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205035.944518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205035.944519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205035.944576:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205035.944581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205035.944585:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205036.264492:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301163009, at 4301163520 00000100:00000040:1.0:1788205036.264492:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301163009, at 4301163520 00000100:00000001:3.0:1788205036.264492:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205036.264500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205036.264504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205036.264510:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00100000:3.0:1788205036.264510:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205036.264512:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff90c0. 00000100:00000001:3.0:1788205036.264516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205036.264522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.264525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205036.264529:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.264532:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.712423:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205036.712431:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205036.712433:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205036.712438:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205036.712441:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205036.712444:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205036.712446:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205036.712448:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205036.712450:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776428:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205036.776438:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205036.776441:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205036.776449:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205036.776455:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205036.776460:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.776464:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205036.776467:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.776471:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.776498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205036.776503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205036.776505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205036.776509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:3.0:1788205036.776510:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205036.776512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205036.776515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205036.776518:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205036.776519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205036.776523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.776523:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205036.776528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205036.776529:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205036.776535:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205036.776545:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6491 00000400:00000200:3.0:1788205036.776556:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205036.776571:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205036.776575:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205036.776579:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776582:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.776585:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776587:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205036.776591:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205036.776594:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205036.776597:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205036.776601:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205036.776603:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205036.776606:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205036.776609:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776611:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.776613:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776614:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205036.776618:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205036.776621:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776624:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205036.776628:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205036.776632:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205036.776635:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776637:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205036.776640:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205036.776649:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3b80. 00000020:00000040:3.0:1788205036.776652:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205036.776657:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205036.776658:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776661:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205036.776663:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205036.776667:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205036.776675:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23000. 00000100:00000001:3.0:1788205036.776681:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205036.776690:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205036.776698:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205036.776702:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205036.776706:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c10db33c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205036.776723:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205036.776726:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205036.776730:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205036.776732:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205036.776733:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205036.776737:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205036.776739:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6491 00000400:00000200:3.0:1788205036.776743:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205036.776746:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205036.776748:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788205036.776749:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205036.776752:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205036.776754:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205036.776755:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205036.776756:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205036.776757:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776759:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205036.776761:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:0.0:1788205036.776763:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205036.776763:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205036.776764:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205036.776767:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205036.776769:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205036.776771:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205036.776772:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205036.776774:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205036.776774:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776776:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205036.776778:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1788205036.776778:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776780:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:0.0:1788205036.776781:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1788205036.776782:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:0.0:1788205036.776785:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205036.776785:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776787:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205036.776788:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205922432:192.168.201.151@tcp:250:kworker.0 02000000:00000001:3.0:1788205036.776789:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205036.776793:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205036.776796:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205036.776798:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205036.776798:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776800:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1788205036.776802:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.776805:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1788205036.776806:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000010:0.0:1788205036.776811:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205036.776813:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205036.776814:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776816:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205036.776819:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0e00. 00000400:00000200:0.0:1788205036.776820:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:3.0:1788205036.776821:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205036.776824:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1788205036.776825:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 02000000:00000001:3.0:1788205036.776826:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776827:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205036.776828:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205036.776830:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205036.776833:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 02000000:00000010:3.0:1788205036.776833:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21400. 00000100:00000001:3.0:1788205036.776836:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205036.776841:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1788205036.776843:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922432, portal 25 00000100:00000001:0.0:1788205036.776846:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205036.776848:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000001:3.0:1788205036.776849:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205036.776851:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205036.776854:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205036.776854:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000233989f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.776863:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205036.776863:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205036.776865:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1788205036.776867:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205036.776869:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00080000:3.0:1788205036.776869:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205036.776870:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205036.776872:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1788205036.776873:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205922432, offset 0 00000100:00000001:3.0:1788205036.776875:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205036.776878:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6491 00000400:00000200:0.0:1788205036.776880:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205036.776881:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205036.776883:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205036.776886:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205036.776888:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.776907:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205036.776907:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.776909:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776911:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205036.776913:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1788205036.776914:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:3.0:1788205036.776915:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205036.776916:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205036.776919:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:0.0:1788205036.776921:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1788205036.776921:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205036.776923:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788205036.776925:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.776927:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:3.0:1788205036.776927:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.776929:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.776931:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205036.776932:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205036.776934:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:3.0:1788205036.776934:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205036.776937:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:0.0:1788205036.776938:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59062:59062:256:4294967295] 192.168.201.151@tcp LPNI seq info [59042:59042:8:4294967295] 00000100:00000001:3.0:1788205036.776939:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205036.776941:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1788205036.776943:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788205036.776944:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205036.776946:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776947:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205036.776950:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205036.776952:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205036.776954:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:3.0:1788205036.776954:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205036.776958:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2680. 00000020:00000040:3.0:1788205036.776959:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:0.0:1788205036.776962:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788205036.776962:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205036.776963:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.776965:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205036.776966:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:0.0:1788205036.776968:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 02000000:00000001:3.0:1788205036.776968:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205036.776971:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20400. 00000100:00000001:3.0:1788205036.776974:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205036.776976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:3.0:1788205036.776978:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1788205036.776980:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1788205036.776985:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1788205036.776986:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788205036.776987:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205036.776990:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bb9116c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788205036.776991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788205036.776996:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788205036.776998:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205036.777000:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000100:00080000:3.0:1788205036.777001:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205036.777004:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205036.777005:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205036.777007:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1788205036.777010:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1788205036.777010:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205036.777012:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6491 00000400:00000200:3.0:1788205036.777015:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777016:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205036.777018:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205036.777021:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1788205036.777023:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000100:00000040:3.0:1788205036.777024:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:0.0:1788205036.777025:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:3.0:1788205036.777026:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.777028:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205036.777029:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205036.777031:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1788205036.777031:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205036.777033:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205036.777034:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:0.0:1788205036.777035:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205036.777037:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205036.777039:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205036.777041:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205036.777044:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205036.777046:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205036.777046:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205036.777048:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205036.777049:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205036.777050:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205036.777052:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205036.777055:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205036.777057:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777058:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205036.777058:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205036.777060:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1788205036.777061:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205036.777062:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.777064:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205036.777066:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:0.0:1788205036.777067:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205036.777068:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.777070:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205036.777071:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:3.0:1788205036.777073:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3800. 00000100:00000001:0.0:1788205036.777074:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301168129 00000020:00000040:3.0:1788205036.777075:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205036.777078:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205036.777079:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205036.777080:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205036.777080:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205036.777082:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205036.777083:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205036.777086:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22800. 00000100:00000001:0.0:1788205036.777089:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205036.777089:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205036.777093:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205036.777097:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301168129 00000100:00000010:0.0:1788205036.777100:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9360. 00000100:00000001:3.0:1788205036.777101:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205036.777103:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205036.777103:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777107:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788205036.777107:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3172b36] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777111:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777113:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205036.777114:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205036.777117:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205036/real 1788205036] req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205036.777119:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6501) 00000400:00000001:0.0:1788205036.777127:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205036.777129:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205036.777131:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205036.777134:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205036.777135:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205036.777138:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205036.777142:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205036.777151:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777158:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205036.777159:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777162:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777164:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205036.777168:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777175:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205036.777178:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777183:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205036.777185:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777186:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777186:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777188:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205036.777190:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205036.777191:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205036.777194:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205036.777195:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205036.777198:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205036.777203:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205922432:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205036.777207:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777209:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.777211:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205922432/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777223:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205036.777226:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205036.777229:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205036.777232:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205036.777234:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205036.777236:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205036.777238:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205036.777241:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:0.0:1788205036.777245:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205036.777247:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205036.777249:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:0.0:1788205036.777252:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205036.777253:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777256:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777264:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205036.777265:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777266:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205036.777267:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205036.777268:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205036.777271:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777275:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205036.777276:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777278:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205036.777279:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205036.777280:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205036.777282:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922560:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205036.777284:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205036.777286:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205036.777287:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777288:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205036.777290:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205036.777291:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205036.777294:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205036.777296:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205036.777299:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205036.777301:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922560, portal 10 00000100:00000001:0.0:1788205036.777302:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205036.777303:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:0.0:1788205036.777308:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777312:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205036.777313:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205036.777314:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205036.777316:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205922560, offset 0 00000400:00000200:0.0:1788205036.777318:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777321:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777323:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205036.777325:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205036.777327:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205036.777328:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205036.777330:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777332:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59063:59063:256:4294967295] 192.168.201.151@tcp LPNI seq info [59043:59043:8:4294967295] 00000400:00000200:0.0:1788205036.777335:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777339:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205036.777342:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777344:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205036.777347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205036.777348:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788205036.777351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205036.777384:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788205036.777387:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777389:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205036.777392:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205036.777395:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205036.777398:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205036.777400:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205036.777403:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205036.777407:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777415:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.777418:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777425:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777426:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205036.777430:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205036.777434:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205036.777438:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205036.777440:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301164187 00000100:00000001:0.0:1788205036.777443:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.777445:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301164187 00000100:00000001:0.0:1788205036.777447:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.777449:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205036.777451:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777452:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205036.777456:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205036/real 1788205036] req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205036.777464:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205036.777466:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205036.777467:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205036.777470:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205036.777472:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205036.777474:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205036.777477:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205036.777486:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777498:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205036.777499:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777501:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777503:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205036.777506:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777513:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205036.777517:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777524:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205036.777526:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777527:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777529:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777531:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205036.777533:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205036.777535:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205036.777538:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205036.777540:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205036.777544:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205036.777551:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922560:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205036.777555:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205036.777558:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777564:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205036.777565:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777568:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205036.777580:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205036.777585:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205036.777588:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205036.777592:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205036.777593:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205036.777595:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777597:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205036.777600:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205036.777602:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205036.777606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205036.777609:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205036.777612:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205036.777615:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922688, portal 4 00000100:00000001:0.0:1788205036.777617:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205036.777618:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:0.0:1788205036.777624:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777631:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205036.777634:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205036.777636:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205036.777638:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205922688, offset 0 00000400:00000200:0.0:1788205036.777643:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777646:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777650:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205036.777653:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205036.777655:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205036.777657:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205036.777660:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777663:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59064:59064:256:4294967295] 192.168.201.151@tcp LPNI seq info [59044:59044:8:4294967295] 00000400:00000200:0.0:1788205036.777666:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777672:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205036.777677:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777680:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205036.777684:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205036.777687:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788205036.777691:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205036.777694:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788205036.777698:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777700:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205036.777703:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205036.777706:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205036.777709:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205036.777711:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205036.777714:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205036.777717:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777725:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.777728:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777733:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777734:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205036.777738:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205036.777741:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205036.777744:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205036.777746:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301164073 00000100:00000001:0.0:1788205036.777748:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.777751:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301164073 00000100:00000001:0.0:1788205036.777753:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.777755:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205036.777757:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777759:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205036.777763:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205036/real 1788205036] req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205036.777770:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205036.777772:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205036.777773:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205036.777776:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205036.777777:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205036.777779:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205036.777782:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205036.777790:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777797:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205036.777798:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777800:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777801:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205036.777805:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777811:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205036.777814:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777821:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205036.777823:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777823:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.777824:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.777825:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205036.777827:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205036.777829:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205036.777831:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205036.777833:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205036.777836:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205036.777841:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205036.777843:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205036.777844:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777848:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205036.777849:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777850:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205036.777852:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205036.777853:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205036.777855:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205036.777856:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205036.777857:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205036.777858:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205036.777859:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205036.777862:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205036.777863:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205036.777865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205036.777867:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205036.777869:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205036.777871:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922816, portal 4 00000100:00000001:0.0:1788205036.777872:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205036.777873:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:0.0:1788205036.777877:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777881:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205036.777882:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205036.777883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000200:0.0:1788205036.777885:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205922816, offset 0 00000400:00000200:0.0:1788205036.777888:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777907:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777910:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205036.777913:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205036.777915:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205036.777917:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205036.777919:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205036.777922:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59065:59065:256:4294967295] 192.168.201.151@tcp LPNI seq info [59045:59045:8:4294967295] 00000400:00000200:0.0:1788205036.777926:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205036.777932:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205036.777937:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777940:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205036.777944:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205036.777947:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788205036.777951:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205036.777953:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788205036.777957:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205036.777959:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:0.0:1788205036.777962:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205036.777965:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205036.777968:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:0.0:1788205036.777970:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:0.0:1788205036.777973:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205036.777976:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777983:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.777984:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.777988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.777989:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205036.777992:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205036.777994:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205036.777996:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205036.777997:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301164079 00000100:00000001:0.0:1788205036.777999:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.778000:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301164079 00000100:00000001:0.0:1788205036.778002:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301168129 00000100:00000001:0.0:1788205036.778003:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205036.778004:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.778005:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205036.778007:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205036/real 1788205036] req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205036.778011:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205036.778012:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205036.778014:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205036.778016:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205036.778017:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205036.778019:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205036.778022:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205036.778030:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778036:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205036.778038:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778040:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.778041:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205036.778045:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778051:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205036.778055:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778060:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205036.778063:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778064:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205036.778065:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778067:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205036.778069:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205036.778072:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205036.778075:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205036.778077:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205036.778080:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205036.778088:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205922816:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205036.778091:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778093:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.778095:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205922560/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778102:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205036.778104:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205036.778105:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205036.778107:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205036.778109:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205036.778110:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205036.778112:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205036.778114:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:0.0:1788205036.778116:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205036.778118:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205036.778119:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:0.0:1788205036.778122:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205036.778124:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778125:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.778128:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205922688/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778140:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205036.778141:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205036.778142:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205036.778144:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205036.778146:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205036.778147:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205036.778149:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205036.778150:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:0.0:1788205036.778153:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205036.778154:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205036.778155:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:0.0:1788205036.778157:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205036.778159:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778160:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205036.778164:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205922816/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205036.778170:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205036.778171:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205036.778172:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205036.778175:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205036.778176:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205036.778177:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205036.778179:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205036.778180:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:0.0:1788205036.778182:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205036.778183:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205036.778184:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:0.0:1788205036.778186:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205036.778187:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778189:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205036.778192:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205036.778194:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205036.778195:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205036.778197:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205036.968529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205036.968537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205036.968543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205036.968544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205036.968546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205036.968547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205036.968549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205036.968550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205036.968566:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205036.968568:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205036.968570:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205037.288468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205037.288475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205037.288476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205037.288481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205037.288484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205037.288487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205037.288490:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205037.288493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205037.288496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205037.736478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205037.736484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205037.736486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205037.736490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205037.736493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205037.736495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.736497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205037.736498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.736500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205037.800469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205037.800469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205037.800476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205037.800476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205037.800478:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205037.800478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205037.800483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205037.800483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205037.800486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205037.800486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205037.800490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205037.800490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.800492:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205037.800494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.800497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205037.800498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205037.800499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205037.800501:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205037.992463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205037.992471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205037.992477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205037.992478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205037.992486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205037.992487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205037.992490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205037.992491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205037.992604:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205037.992607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205037.992610:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205038.312507:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205038.312514:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205038.312516:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205038.312522:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205038.312525:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205038.312529:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205038.312532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205038.312535:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205038.312538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.760483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205038.760490:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205038.760491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205038.760497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205038.760500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205038.760504:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.760507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.760510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.760513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.824475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205038.824475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205038.824484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205038.824486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.824490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205038.824492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205038.824494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.824497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788205038.824499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205038.824501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.824504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205038.824506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.824507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205038.824510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.824515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205038.824515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.824518:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.824520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205039.016528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205039.016534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205039.016538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205039.016540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205039.016542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205039.016544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205039.016547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205039.016548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205039.016564:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205039.016566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205039.016569:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205039.336481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205039.336490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205039.336492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205039.336498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205039.336501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205039.336505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205039.336508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205039.336510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205039.336513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.784494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205039.784501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205039.784504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205039.784509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205039.784513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205039.784516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205039.784518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205039.784521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205039.784524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.848523:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205039.848533:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205039.848535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205039.848542:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205039.848547:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205039.848553:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205039.848556:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205039.848559:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205039.848563:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205039.848631:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205039.848634:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205039.848635:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205039.848638:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205039.848641:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205039.848642:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205039.848645:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205039.848646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205039.848648:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205040.040523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205040.040531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205040.040537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205040.040538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205040.040541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205040.040542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205040.040546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205040.040548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205040.040616:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205040.040620:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205040.040623:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205040.360471:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205040.360478:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205040.360480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205040.360485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205040.360489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205040.360493:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205040.360495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205040.360498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205040.360500:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.808475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205040.808482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205040.808484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205040.808488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205040.808492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205040.808495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.808498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205040.808499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.808501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205040.872513:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205040.872514:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205040.872522:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205040.872524:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205040.872525:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.872529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205040.872532:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205040.872538:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205040.872538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205040.872542:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.872545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205040.872546:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205040.872552:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.872554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205040.872555:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.872572:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205040.872575:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.872579:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064643:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205041.064649:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064653:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205041.064656:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205041.064659:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205041.064662:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064664:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205041.064666:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064668:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064670:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064672:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205041.064673:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205041.064675:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205041.064675:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205041.064689:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205041.064691:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205041.064693:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205041.384486:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301168129, at 4301168640 00000100:00000040:1.0:1788205041.384486:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301168129, at 4301168640 00000100:00000001:3.0:1788205041.384486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205041.384493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205041.384497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205041.384501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205041.384502:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000010:1.0:1788205041.384504:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9360. 00000100:00000001:3.0:1788205041.384504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205041.384509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205041.384512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205041.384515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205041.384517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205041.832471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205041.832481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205041.832484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205041.832490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205041.832495:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205041.832500:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.832503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205041.832507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.832510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205041.896444:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205041.896452:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205041.896454:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205041.896460:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:3.0:1788205041.896463:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205041.896465:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788205041.896469:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205041.896472:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205041.896476:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205041.896477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.896482:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205041.896484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205041.896484:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205041.896486:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.896491:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205041.896491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205041.896495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205041.896497:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6496 00000100:00100000:2.0:1788205041.896501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205041.896505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205041.896508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1788205041.896510:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205041.896512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205041.896516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205041.896519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205041.896522:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205041.896526:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205041.896528:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896531:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896534:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896535:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205041.896539:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205041.896541:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205041.896543:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205041.896546:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205041.896547:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205041.896549:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205041.896551:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896552:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896553:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896554:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205041.896556:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205041.896558:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896560:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205041.896562:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205041.896565:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205041.896567:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896576:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205041.896580:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205041.896590:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1c00. 00000020:00000040:3.0:1788205041.896592:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205041.896596:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205041.896597:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896599:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205041.896602:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205041.896605:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205041.896612:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23c00. 00000100:00000001:3.0:1788205041.896619:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205041.896627:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205041.896638:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205041.896642:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205041.896646:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004207d831] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205041.896661:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205041.896664:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205041.896666:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205041.896667:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205041.896669:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205041.896673:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205041.896674:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6496 00000400:00000200:3.0:1788205041.896676:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205041.896678:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205041.896680:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205041.896682:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896683:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896684:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896685:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205041.896686:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205041.896687:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205041.896688:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205041.896690:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205041.896692:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205041.896694:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205041.896696:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896698:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896699:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896700:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205041.896701:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788205041.896703:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896707:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.896708:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788205041.896708:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205041.896714:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788205041.896714:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205041.896718:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.896719:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205041.896719:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205041.896726:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1788205041.896729:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788205041.896729:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896733:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205041.896740:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3800. 00000100:00000001:0.0:1788205041.896744:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1788205041.896744:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205041.896746:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788205041.896748:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205041.896749:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896754:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205041.896755:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205041.896757:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205041.896757:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.896762:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1788205041.896762:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20c00. 00000100:00000001:3.0:1788205041.896764:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.896768:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205041.896772:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205041.896777:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205041.896778:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205041.896780:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3172b36] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205041.896786:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205922944:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205041.896787:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205041.896790:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205041.896794:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205041.896796:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788205041.896796:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205041.896798:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1788205041.896819:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205041.896823:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896826:0:12771:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1788205041.896827:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788205041.896830:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6496 00000400:00000200:3.0:1788205041.896834:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:0.0:1788205041.896835:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 00000400:00000200:3.0:1788205041.896837:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 02000000:00000001:0.0:1788205041.896840:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205041.896840:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205041.896845:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896847:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896850:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205041.896852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205041.896852:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205041.896855:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205041.896857:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205041.896858:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:0.0:1788205041.896860:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000020:00000040:3.0:1788205041.896860:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788205041.896863:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205041.896866:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788205041.896868:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896869:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.896870:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896871:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000010:0.0:1788205041.896872:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000020:00000040:3.0:1788205041.896873:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205041.896878:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896880:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205041.896881:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205041.896883:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1788205041.896884:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205922944, portal 25 02000000:00000001:3.0:1788205041.896884:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896889:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205041.896890:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788205041.896891:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:0.0:1788205041.896893:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300828672 : -113511408722944 : ffff98c310ce1c00) 00000008:00000001:3.0:1788205041.896894:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896899:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205041.896902:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2680. 00000100:00000040:0.0:1788205041.896906:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205041.896911:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205041.896916:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205041.896919:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.896921:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788205041.896922:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788205041.896923:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205041.896925:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205041.896927:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22000. 00000100:00000001:3.0:1788205041.896930:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205041.896934:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1788205041.896937:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205041.896942:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217990. 00000100:00000001:3.0:1788205041.896943:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205041.896946:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205041.896948:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205922944, offset 0 00000100:00000040:3.0:1788205041.896951:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bb9116c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205041.896957:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205041.896960:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:3.0:1788205041.896960:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205041.896965:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205041.896966:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205041.896967:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205041.896969:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205041.896971:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6496 00000400:00000200:0.0:1788205041.896975:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205041.896978:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205041.896981:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205041.896982:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1788205041.896992:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:3.0:1788205041.896992:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.896998:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205041.897001:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897002:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205041.897005:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205041.897009:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1788205041.897010:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:3.0:1788205041.897012:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205041.897015:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:0.0:1788205041.897018:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1788205041.897019:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:0.0:1788205041.897023:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:3.0:1788205041.897024:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205041.897029:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:0.0:1788205041.897031:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59066:59066:256:4294967295] 192.168.201.151@tcp LPNI seq info [59046:59046:8:4294967295] 00000100:00000040:3.0:1788205041.897036:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.897039:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205041.897042:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:3.0:1788205041.897042:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205041.897047:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205041.897049:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205041.897051:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205041.897052:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205041.897054:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205041.897058:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205041.897060:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 02000000:00000001:3.0:1788205041.897061:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.897063:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205041.897065:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788205041.897066:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.897068:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205041.897071:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0e00. 00000020:00000040:3.0:1788205041.897073:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205041.897076:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:0.0:1788205041.897078:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:3.0:1788205041.897078:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.897083:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205041.897084:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205041.897085:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205041.897088:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 02000000:00000010:3.0:1788205041.897089:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22400. 00000100:00000001:3.0:1788205041.897093:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205041.897100:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:3.0:1788205041.897100:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205041.897110:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205041.897111:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205041.897115:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000233989f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788205041.897115:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788205041.897124:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205041.897127:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6506) 00000100:00000001:0.0:1788205041.897134:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205041.897140:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205041.897144:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301173249 00000100:00000010:0.0:1788205041.897149:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205041.897158:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205041.897164:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301173249 00000100:00000010:0.0:1788205041.897168:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9300. 00000100:00000001:0.0:1788205041.897171:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205041.897180:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205041.897184:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897187:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897190:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205041.897197:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205041.897199:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205041.897201:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205041.897203:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205041.897205:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205041.897209:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897217:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205041.897219:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897221:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205041.897223:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205041.897226:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205041.897229:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923072:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205041.897233:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205041.897234:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205041.897236:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897238:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205041.897242:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205041.897244:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205041.897247:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205041.897251:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205041.897255:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217cc0. 00000100:00000200:0.0:1788205041.897259:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923072, portal 10 00000100:00000001:0.0:1788205041.897269:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205041.897273:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300835840 : -113511408715776 : ffff98c310ce3800) 00000100:00000040:0.0:1788205041.897282:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897290:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205041.897292:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205041.897296:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217ee0. 00000100:00000200:0.0:1788205041.897299:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205923072, offset 0 00000400:00000200:0.0:1788205041.897305:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897308:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897314:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205041.897318:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205041.897320:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205041.897322:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205041.897325:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897328:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59067:59067:255:4294967295] 192.168.201.151@tcp LPNI seq info [59047:59047:7:4294967295] 00000400:00000200:0.0:1788205041.897333:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897339:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205041.897343:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205041.897347:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205041.897351:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205041.897355:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205041.897358:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205041.897362:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205041.897403:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205041.897405:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301169307 00000100:00000001:0.0:1788205041.897408:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897410:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301169307 00000100:00000001:0.0:1788205041.897412:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897414:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205041.897417:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897419:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205041.897423:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897430:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205041.897432:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897433:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205041.897435:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205041.897437:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205041.897440:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205041.897443:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205041.897445:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205041.897446:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897448:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205041.897452:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205041.897454:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205041.897458:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205041.897460:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da268. 00000400:00000010:0.0:1788205041.897464:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217440. 00000100:00000200:0.0:1788205041.897467:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923200, portal 4 00000100:00000001:0.0:1788205041.897469:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205041.897470:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300831360 : -113511408720256 : ffff98c310ce2680) 00000100:00000040:0.0:1788205041.897476:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897483:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205041.897486:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205041.897488:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217660. 00000100:00000200:0.0:1788205041.897491:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205923200, offset 0 00000400:00000200:0.0:1788205041.897495:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897499:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897502:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205041.897505:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205041.897507:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205041.897510:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205041.897512:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897515:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59068:59068:254:4294967295] 192.168.201.151@tcp LPNI seq info [59048:59048:6:4294967295] 00000400:00000200:0.0:1788205041.897519:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897525:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205041.897529:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205041.897533:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054afb00. 00000800:00000200:0.0:1788205041.897537:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205041.897541:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205041.897544:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205041.897547:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205041.897550:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205041.897552:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301169193 00000100:00000001:0.0:1788205041.897554:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897557:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301169193 00000100:00000001:0.0:1788205041.897559:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897561:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205041.897563:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897565:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205041.897568:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897575:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205041.897577:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897579:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205041.897581:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205041.897582:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205041.897585:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205041.897589:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205041.897591:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205041.897592:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205041.897593:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205041.897597:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205041.897599:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205041.897602:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205041.897604:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da3c8. 00000400:00000010:0.0:1788205041.897607:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205041.897610:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923328, portal 4 00000100:00000001:0.0:1788205041.897612:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205041.897613:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825088 : -113511408726528 : ffff98c310ce0e00) 00000100:00000040:0.0:1788205041.897619:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.897625:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205041.897627:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205041.897629:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217bb0. 00000100:00000200:0.0:1788205041.897632:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205923328, offset 0 00000400:00000200:0.0:1788205041.897635:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897638:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897642:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205041.897644:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205041.897646:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205041.897648:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205041.897650:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205041.897653:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59069:59069:253:4294967295] 192.168.201.151@tcp LPNI seq info [59049:59049:5:4294967295] 00000400:00000200:0.0:1788205041.897656:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205041.897661:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205041.897664:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205041.897668:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788205041.897671:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205041.897675:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205041.897678:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205041.897680:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205041.897683:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205041.897685:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301169199 00000100:00000001:0.0:1788205041.897687:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897689:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301169199 00000100:00000001:0.0:1788205041.897691:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301173249 00000100:00000001:0.0:1788205041.897693:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205041.897694:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897696:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897698:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205041.897701:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205041.897703:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205041.897705:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205041.897706:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205041.897708:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.897710:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205042.088477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205042.088486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205042.088493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205042.088494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205042.088499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205042.088500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205042.088504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205042.088505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205042.088559:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205042.088563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205042.088566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205042.408496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205042.408506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205042.408509:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205042.408516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205042.408521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205042.408526:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205042.408529:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205042.408532:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205042.408536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205042.856494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205042.856503:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205042.856504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205042.856510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205042.856513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205042.856517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205042.856520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205042.856522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205042.856524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205042.920680:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205042.920691:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205042.920693:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205042.920695:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205042.920701:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205042.920706:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205042.920710:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205042.920713:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205042.920713:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205042.920718:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205042.920721:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205042.920724:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205042.920748:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205042.920752:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205042.920755:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205042.920757:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205042.920759:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205042.920761:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205043.112549:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205043.112551:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205043.112554:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112559:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205043.112561:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112562:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112564:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205043.112572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205043.112576:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205043.112577:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205043.112580:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112581:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205043.112582:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112584:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112585:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205043.112588:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205043.112589:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205043.112590:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112591:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205043.112592:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112593:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205043.112594:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205043.112609:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205043.112611:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205043.112613:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205043.432444:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205043.432464:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205043.432467:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.432475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205043.432479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205043.432484:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205043.432490:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205043.432495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205043.432498:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.880460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205043.880470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205043.880473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.880480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205043.880485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205043.880490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.880493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.880496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.880499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.944430:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205043.944438:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205043.944441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.944445:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205043.944449:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205043.944453:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.944456:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.944460:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.944463:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205043.944514:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205043.944518:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205043.944520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205043.944523:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205043.944525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205043.944528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205043.944529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205043.944531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205043.944533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205044.136514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205044.136524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205044.136533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205044.136535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205044.136538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205044.136540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205044.136543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205044.136545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205044.136617:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205044.136625:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205044.136631:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205044.456521:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205044.456530:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205044.456532:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205044.456539:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205044.456554:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205044.456558:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205044.456561:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205044.456564:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205044.456567:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205044.904468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205044.904475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205044.904477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205044.904483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205044.904486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205044.904490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205044.904492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205044.904495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205044.904498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205044.968495:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205044.968503:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205044.968505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205044.968510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205044.968513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205044.968517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205044.968520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205044.968522:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205044.968525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205044.969145:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205044.969148:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205044.969150:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205044.969153:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205044.969155:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205044.969156:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205044.969158:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205044.969160:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205044.969162:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1788205044.970082:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:3.0:1788205044.970120:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1788205044.970129:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205044.970133:0:2336:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:3.0:1788205044.970138:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205044.970142:0:2336:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1788205044.970147:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:3.0:1788205044.970151:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:3.0:1788205044.970157:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788205044.970162:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788205044.970168:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217990 00000400:00000010:3.0:1788205044.970170:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217990. 00000100:00000001:3.0:1788205044.970175:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205044.970185:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970211:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205044.970223:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970231:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.970233:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788205044.970241:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:3.0:1788205044.970243:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:3.0:1788205044.970246:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788205044.970249:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788205044.970252:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217ee0 00000400:00000010:3.0:1788205044.970255:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217ee0. 00000100:00000001:3.0:1788205044.970257:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205044.970260:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970268:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205044.970271:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970277:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.970279:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788205044.970283:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:3.0:1788205044.970285:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054afb00. 00000400:00000200:3.0:1788205044.970288:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788205044.970291:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788205044.970293:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217660 00000400:00000010:3.0:1788205044.970295:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217660. 00000100:00000001:3.0:1788205044.970297:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205044.970301:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970308:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205044.970311:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970317:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.970318:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788205044.970322:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:3.0:1788205044.970324:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:3.0:1788205044.970326:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788205044.970329:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788205044.970332:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217bb0 00000400:00000010:3.0:1788205044.970333:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217bb0. 00000100:00000001:3.0:1788205044.970336:0:2336:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205044.970340:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970347:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205044.970349:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205044.970356:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.970358:0:2336:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205044.971405:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205044.971409:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205044.971412:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205044.971420:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205041/real 1788205044] req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205044.971433:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205044.971435:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205044.971438:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205044.971442:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217cc0 00000400:00000010:0.0:1788205044.971444:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217cc0. 00000100:00000001:0.0:1788205044.971447:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205044.971450:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205044.971458:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971466:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205044.971467:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971470:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971472:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205044.971476:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971483:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205044.971487:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971494:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205044.971498:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971499:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971501:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971503:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205044.971506:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205044.971509:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205044.971513:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205044.971516:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205044.971520:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205044.971528:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923072:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205044.971532:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205044.971536:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205041/real 1788205044] req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205044.971543:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205044.971544:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205044.971547:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da268. 00000400:00000200:0.0:1788205044.971549:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217440 00000400:00000010:0.0:1788205044.971550:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217440. 00000100:00000001:0.0:1788205044.971552:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205044.971555:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205044.971563:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971569:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205044.971571:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971572:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971574:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205044.971577:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971583:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205044.971587:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971594:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205044.971596:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971597:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971598:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971599:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205044.971601:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205044.971603:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205044.971606:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205044.971608:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205044.971612:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205044.971619:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205044.971623:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205044.971627:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205041/real 1788205044] req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205044.971634:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205044.971635:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205044.971636:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da3c8. 00000400:00000200:0.0:1788205044.971638:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205044.971639:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205044.971641:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205044.971643:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205044.971656:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971662:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205044.971663:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971665:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971666:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205044.971669:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971674:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205044.971678:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971685:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205044.971687:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971688:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971690:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971691:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205044.971693:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205044.971695:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205044.971697:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205044.971699:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205044.971702:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205044.971710:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923328:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205044.971712:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205044.971716:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205041/real 1788205044] req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205044.971732:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205044.971733:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205044.971734:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205044.971736:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205044.971738:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205044.971740:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205044.971743:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205044.971768:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971774:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205044.971776:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971778:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971779:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205044.971782:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971788:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205044.971792:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971798:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205044.971800:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971801:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205044.971802:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971804:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205044.971805:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205044.971808:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205044.971811:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205044.971813:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205044.971816:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205044.971824:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205922944:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205044.971828:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971830:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205044.971834:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3800 x1875062205923072/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971840:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205044.971843:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205044.971846:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205044.971850:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205044.971852:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205044.971854:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205044.971856:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205044.971859:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:0.0:1788205044.971862:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205044.971864:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205044.971866:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3800. 00000100:00000001:0.0:1788205044.971870:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205044.971871:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971872:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205044.971876:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2680 x1875062205923200/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971881:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205044.971883:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205044.971884:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205044.971886:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205044.971888:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205044.971889:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205044.971890:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205044.971892:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:0.0:1788205044.971894:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205044.971895:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205044.971896:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2680. 00000100:00000001:0.0:1788205044.971899:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205044.971900:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971901:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205044.971904:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0e00 x1875062205923328/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971909:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205044.971910:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205044.971912:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205044.971914:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205044.971915:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205044.971916:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205044.971917:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205044.971918:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:0.0:1788205044.971921:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205044.971922:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205044.971923:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0e00. 00000100:00000001:0.0:1788205044.971925:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205044.971927:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971928:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205044.971931:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1c00 x1875062205922944/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205044.971936:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205044.971937:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205044.971938:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205044.971940:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205044.971941:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205044.971942:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205044.971944:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205044.971945:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:0.0:1788205044.971947:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205044.971948:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205044.971949:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1c00. 00000100:00000001:0.0:1788205044.971951:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205044.971952:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971954:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205044.971960:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205044.971962:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205044.971964:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205044.971965:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205045.160526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205045.160532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205045.160537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205045.160538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205045.160541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205045.160542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205045.160545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205045.160546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205045.160562:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205045.160564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205045.160566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205045.480490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205045.480497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205045.480499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205045.480505:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205045.480508:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205045.480512:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205045.480514:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205045.480517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205045.480520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205045.928483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205045.928497:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205045.928499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205045.928504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205045.928507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205045.928511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205045.928514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205045.928516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205045.928520:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205045.992467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205045.992473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205045.992482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205045.992487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205045.992490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205045.992494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205045.992496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205045.992499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205045.992502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205045.992510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205045.992513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205045.992514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205045.992516:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205045.992518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205045.992519:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.992521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205045.992522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.992524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205046.184472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205046.184479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205046.184486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205046.184488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205046.184491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205046.184493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205046.184501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205046.184502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205046.184574:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205046.184578:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205046.184581:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205046.504462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788205046.504463:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301173249, at 4301173760 00000100:00000040:1.0:1788205046.504463:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301173249, at 4301173760 00000100:00000010:0.0:1788205046.504475:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205046.504475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1788205046.504477:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9300. 00000100:00000001:3.0:1788205046.504480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205046.504486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205046.504489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205046.504493:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205046.504496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205046.504499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205046.504502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205046.952454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205046.952462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205046.952464:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205046.952469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205046.952473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205046.952477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205046.952479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205046.952482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205046.952485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205047.016492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205047.016500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205047.016503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205047.016508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205047.016511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205047.016515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.016518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205047.016519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.016522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205047.016607:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205047.016611:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205047.016613:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205047.016615:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205047.016618:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205047.016622:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6501 00000400:00000200:3.0:1788205047.016629:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205047.016633:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205047.016635:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205047.016644:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016646:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205047.016648:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016650:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205047.016652:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205047.016654:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205047.016656:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205047.016659:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205047.016660:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205047.016661:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205047.016663:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016664:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205047.016666:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016667:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205047.016669:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205047.016671:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016673:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205047.016675:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205047.016678:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205047.016679:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.016681:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205047.016682:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205047.016696:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3b80. 00000020:00000040:3.0:1788205047.016700:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205047.016704:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205047.016705:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.016708:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205047.016710:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205047.016730:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205047.016738:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20800. 00000100:00000001:3.0:1788205047.016745:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205047.016755:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205047.016763:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205047.016765:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205047.016768:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c10db33c] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205047.016788:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205047.016790:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205047.016793:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205047.016794:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205047.016795:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205047.016797:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205047.016798:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6501 00000400:00000200:3.0:1788205047.016801:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205047.016803:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205047.016805:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205047.016807:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016809:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205047.016811:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016813:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205047.016814:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205047.016815:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205047.016816:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205047.016818:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205047.016819:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205047.016820:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205047.016822:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016824:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205047.016826:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016827:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205047.016828:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788205047.016830:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205047.016832:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.016834:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788205047.016834:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205047.016837:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205047.016838:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205047.016842:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788205047.016842:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.016846:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205047.016850:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1788205047.016851:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788205047.016852:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.016856:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205047.016861:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1500. 00000020:00000040:3.0:1788205047.016864:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:0.0:1788205047.016866:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205047.016868:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 02000000:00000001:3.0:1788205047.016868:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788205047.016870:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205047.016871:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205047.016873:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205047.016875:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205047.016876:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205047.016877:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205047.016878:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205047.016881:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205047.016882:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205047.016883:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 02000000:00000001:0.0:1788205047.016886:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205047.016889:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1788205047.016890:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21800. 00000100:00100000:0.0:1788205047.016893:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205923456:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205047.016893:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205047.016898:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205047.016902:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205047.016902:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205047.016907:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.016908:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205047.016910:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.016912:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.016916:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788205047.016920:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1788205047.016924:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205047.016928:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.016936:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205047.016939:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000100:00000001:3.0:1788205047.016940:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205047.016944:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205047.016945:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205047.016951:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923456, portal 25 00000100:00000040:3.0:1788205047.016951:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bfc2cda] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.016955:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205047.016958:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300836736 : -113511408714880 : ffff98c310ce3b80) 00000100:00000040:0.0:1788205047.016968:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205047.016969:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205047.016973:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205047.016980:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205047.016983:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205047.016986:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205047.016987:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205047.016992:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1788205047.016993:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:3.0:1788205047.016997:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6501 00000400:00000010:0.0:1788205047.016999:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205047.017004:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205923456, offset 0 00000400:00000200:0.0:1788205047.017010:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205047.017018:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.017024:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205047.017029:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205047.017033:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205047.017036:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205047.017039:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205047.017043:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59070:59070:256:4294967295] 192.168.201.151@tcp LPNI seq info [59050:59050:8:4294967295] 00000400:00000200:0.0:1788205047.017047:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.017059:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:3.0:1788205047.017062:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:0.0:1788205047.017068:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.017073:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000400:00000200:3.0:1788205047.017078:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205047.017083:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1788205047.017084:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:3.0:1788205047.017090:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:0.0:1788205047.017091:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1788205047.017095:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205047.017101:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:0.1:1788205047.017102:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788205047.017105:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:0.1:1788205047.017107:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1788205047.017110:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205047.017113:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205047.017116:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1788205047.017117:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000040:3.0:1788205047.017120:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000010:0.0:1788205047.017122:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000020:00000001:3.0:1788205047.017124:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205047.017128:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205047.017130:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1788205047.017133:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.017138:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205047.017139:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1788205047.017143:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.017147:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205047.017148:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000020:00000040:3.0:1788205047.017151:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205047.017155:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:0.0:1788205047.017160:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:3.0:1788205047.017160:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205047.017165:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205047.017168:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1788205047.017169:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205047.017173:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205047.017177:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205047.017177:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205047.017181:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205047.017187:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.017190:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017195:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000010:3.0:1788205047.017197:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0700. 00000100:00000040:0.0:1788205047.017201:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205047.017202:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205047.017207:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205047.017209:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.017213:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205047.017216:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788205047.017219:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205047.017220:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017222:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:3.0:1788205047.017226:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23400. 00000100:00000001:0.0:1788205047.017230:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.017231:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.017237:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788205047.017240:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205047.017242:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301178369 00000100:00000010:0.0:1788205047.017250:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205047.017252:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205047.017256:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017258:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1788205047.017261:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b25c632d] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017267:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301178369 00000100:00000001:3.0:1788205047.017272:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205047.017273:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9960. 00000100:00080000:3.0:1788205047.017277:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205047.017280:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788205047.017283:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205047.017285:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205047.017286:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205047.017290:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205047.017290:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205047.017295:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205047.017296:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205047.017300:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6501 00000100:00100000:0.0:1788205047.017304:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205047.017308:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205047.017314:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205047.017319:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1788205047.017323:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1788205047.017324:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000001:0.0:1788205047.017327:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788205047.017328:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1788205047.017331:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000100:00000040:3.0:1788205047.017332:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.017337:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205047.017338:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000020:00000001:3.0:1788205047.017340:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:0.0:1788205047.017342:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000020:00000040:3.0:1788205047.017344:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205047.017347:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205047.017348:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205047.017350:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205047.017352:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205047.017355:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205047.017358:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205047.017359:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1788205047.017361:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:0.0:1788205047.017403:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205047.017412:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017419:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205047.017420:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017423:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.017425:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205047.017428:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017435:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205047.017439:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017447:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205047.017449:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017451:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.017452:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017454:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205047.017456:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205047.017459:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205047.017462:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205047.017465:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205047.017469:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205047.017476:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205923456:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205047.017481:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017483:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.017486:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3b80 x1875062205923456/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017492:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205047.017494:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205047.017496:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205047.017499:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205047.017501:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205047.017503:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205047.017505:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205047.017508:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:0.0:1788205047.017511:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205047.017513:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205047.017514:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3b80. 00000100:00000001:0.0:1788205047.017517:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205047.017519:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017521:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017529:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205047.017530:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017532:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205047.017534:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205047.017535:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205047.017539:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017545:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205047.017547:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.017549:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205047.017550:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205047.017552:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205047.017555:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923584:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205047.017558:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205047.017560:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205047.017561:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.017563:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205047.017566:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205047.017567:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.017571:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205047.017574:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205047.017577:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205047.017580:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923584, portal 10 00000100:00000001:0.0:1788205047.017582:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205047.017583:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300826880 : -113511408724736 : ffff98c310ce1500) 00000100:00000040:0.0:1788205047.017590:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017596:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205047.017598:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205047.017601:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205047.017603:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205923584, offset 0 00000400:00000200:0.0:1788205047.017607:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205047.017610:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.017614:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205047.017617:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205047.017620:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205047.017622:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205047.017624:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205047.017627:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59071:59071:256:4294967295] 192.168.201.151@tcp LPNI seq info [59051:59051:8:4294967295] 00000400:00000200:0.0:1788205047.017630:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.017641:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205047.017647:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.017650:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205047.017654:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205047.017656:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205047.017660:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205047.017663:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205047.017666:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.017668:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205047.017671:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205047.017674:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205047.017676:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205047.017678:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205047.017681:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205047.017684:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017691:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.017694:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017699:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017701:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205047.017703:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205047.017707:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205047.017710:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205047.017712:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301174426 00000100:00000001:0.0:1788205047.017731:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.017733:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301174426 00000100:00000001:0.0:1788205047.017739:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.017741:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205047.017743:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.017744:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205047.017749:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205047.017757:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205047.017758:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205047.017760:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205047.017763:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205047.017764:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205047.017767:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205047.017769:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205047.017777:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017782:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205047.017783:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017785:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.017786:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205047.017799:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017808:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205047.017812:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017817:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205047.017820:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017822:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.017823:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.017824:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205047.017827:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205047.017829:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205047.017832:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205047.017835:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205047.017838:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205047.017844:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923584:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205047.017848:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205047.017850:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.017856:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205047.017857:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.017859:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205047.017861:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205047.017863:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205047.017866:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205047.017870:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205047.017872:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205047.017874:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.017876:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205047.017880:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205047.017882:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.017885:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205047.017888:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205047.017892:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000001:3.0:1788205047.017898:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:0.0:1788205047.017899:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923712, portal 4 00000020:00000040:3.0:1788205047.017904:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:0.0:1788205047.017905:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205047.017909:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300823296 : -113511408728320 : ffff98c310ce0700) 00000100:00080000:3.0:1788205047.017911:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205047.017916:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205047.017918:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205047.017919:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205047.017924:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205047.017929:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.017933:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205047.017934:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1788205047.017937:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000040:0.0:1788205047.017940:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000008:00000001:3.0:1788205047.017942:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205047.017944:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000001:3.0:1788205047.017946:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205047.017950:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205923712, offset 0 00000100:00000010:3.0:1788205047.017952:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0a80. 00000400:00000200:0.0:1788205047.017958:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000040:3.0:1788205047.017958:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205047.017964:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1788205047.017967:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:3.0:1788205047.017967:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.017971:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205047.017974:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1788205047.017975:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000001:3.0:1788205047.017977:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.017980:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1788205047.017984:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20000. 00000400:00000200:0.0:1788205047.017985:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205047.017990:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205047.017991:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.017995:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00100000:3.0:1788205047.018000:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205047.018001:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59072:59072:256:4294967295] 192.168.201.151@tcp LPNI seq info [59052:59052:8:4294967295] 00000400:00000200:0.0:1788205047.018005:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.018011:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205047.018015:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.018018:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205047.018022:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205047.018024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205047.018028:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205047.018030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205047.018033:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.018035:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205047.018038:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205047.018040:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205047.018043:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205047.018045:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205047.018047:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205047.018050:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018058:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.018061:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018066:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.018068:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205047.018071:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205047.018073:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205047.018076:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205047.018077:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301174313 00000100:00000001:0.0:1788205047.018079:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.018081:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301174313 00000100:00000001:0.0:1788205047.018083:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.018085:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205047.018086:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.018088:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205047.018091:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205047.018097:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205047.018098:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205047.018100:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205047.018102:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205047.018103:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205047.018104:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205047.018107:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205047.018114:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018119:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205047.018120:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018121:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.018123:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205047.018127:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018132:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205047.018135:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018141:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205047.018143:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018145:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.018145:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018147:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205047.018149:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205047.018152:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205047.018155:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205047.018157:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205047.018161:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205047.018168:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205047.018171:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018173:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.018176:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1500 x1875062205923584/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018181:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205047.018183:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205047.018184:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205047.018186:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205047.018188:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205047.018189:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205047.018191:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205047.018193:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:0.0:1788205047.018196:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205047.018197:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205047.018198:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1500. 00000100:00000001:0.0:1788205047.018202:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205047.018203:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018204:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.018207:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0700 x1875062205923712/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.018212:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205047.018214:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205047.018215:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205047.018217:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205047.018218:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205047.018219:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205047.018221:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205047.018223:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:0.0:1788205047.018225:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205047.018226:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205047.018227:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0700. 00000100:00000001:0.0:1788205047.018229:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205047.018230:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018232:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.018236:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205047.018237:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.018239:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205047.018240:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205047.019169:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205047.019172:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205047.019179:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed2b742] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205047.019202:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205047.019208:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6511) 00000100:00000001:0.0:1788205047.019253:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205047.019257:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205047.019259:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205047.019265:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019272:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205047.019274:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.019277:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205047.019279:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205047.019282:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205047.019285:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205047.019288:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205047.019291:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205047.019294:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205047.019295:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205047.019300:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205047.019301:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205047.019305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205047.019309:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205047.019313:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205047.019317:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923840, portal 4 00000100:00000001:0.0:1788205047.019319:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205047.019321:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:0.0:1788205047.019326:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019332:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205047.019334:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205047.019335:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205047.019337:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205923840, offset 0 00000400:00000200:0.0:1788205047.019343:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205047.019428:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.019434:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205047.019438:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205047.019442:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205047.019445:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205047.019447:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205047.019451:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59073:59073:256:4294967295] 192.168.201.151@tcp LPNI seq info [59053:59053:8:4294967295] 00000400:00000200:0.0:1788205047.019456:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205047.019464:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205047.019469:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.019473:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205047.019478:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205047.019481:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205047.019485:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205047.019488:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205047.019491:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205047.019494:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205047.019498:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205047.019501:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205047.019505:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205047.019507:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205047.019510:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205047.019513:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019522:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.019525:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019531:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.019532:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205047.019537:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205047.019540:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205047.019543:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205047.019545:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301174321 00000100:00000001:0.0:1788205047.019548:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.019549:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301174321 00000100:00000001:0.0:1788205047.019551:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301178369 00000100:00000001:0.0:1788205047.019553:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205047.019555:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.019557:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205047.019561:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205047.019569:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205047.019571:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205047.019572:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205047.019574:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205047.019576:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205047.019578:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205047.019581:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205047.019588:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019595:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205047.019596:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019598:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.019600:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205047.019602:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019607:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205047.019609:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019613:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205047.019614:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019615:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205047.019616:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019617:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205047.019619:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205047.019621:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205047.019624:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205047.019625:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205047.019627:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205047.019643:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205923840:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205047.019647:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019649:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205047.019652:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205923840/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205047.019657:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205047.019659:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205047.019661:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205047.019663:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205047.019665:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205047.019667:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205047.019669:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205047.019671:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:0.0:1788205047.019674:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205047.019675:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205047.019676:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:0.0:1788205047.019679:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205047.019680:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019682:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205047.019687:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205047.019689:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205047.019690:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205047.019692:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205047.208507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205047.208512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205047.208515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205047.208516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205047.208518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205047.208519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205047.208521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205047.208521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205047.208534:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205047.208536:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205047.208538:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205047.528484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205047.528491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205047.528492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205047.528497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205047.528499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205047.528503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205047.528505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205047.528506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205047.528508:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205047.976475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205047.976482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205047.976484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205047.976489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205047.976493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205047.976496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.976499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205047.976501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.976504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205048.040608:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205048.040614:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205048.040616:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205048.040621:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205048.040637:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205048.040641:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205048.040644:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205048.040646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205048.040649:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.040665:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205048.040669:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205048.040671:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.040674:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205048.040676:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205048.040678:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.040681:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.040683:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.040685:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205048.233221:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205048.233227:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205048.233230:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205048.233231:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205048.233234:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205048.233235:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205048.233239:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205048.233240:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205048.233747:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205048.233751:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205048.233755:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205048.552467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205048.552474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205048.552476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205048.552482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205048.552484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205048.552487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205048.552489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205048.552491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205048.552493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205049.000452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205049.000459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205049.000461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205049.000467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205049.000471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205049.000474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205049.000476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205049.000478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205049.000480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205049.064490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205049.064497:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205049.064499:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205049.064504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205049.064508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205049.064511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205049.064514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205049.064516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205049.064519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205049.064716:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205049.064720:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205049.064721:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205049.064725:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205049.064727:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205049.064729:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205049.064731:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205049.064733:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205049.064735:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205049.256517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205049.256523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205049.256528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205049.256529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205049.256531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205049.256533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205049.256535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205049.256536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205049.256552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205049.256555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205049.256557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205049.576473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205049.576482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205049.576483:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205049.576488:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205049.576491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205049.576494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205049.576497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205049.576499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205049.576501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.024460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205050.024467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205050.024470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205050.024476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205050.024479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205050.024483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205050.024486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205050.024488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205050.024491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.088432:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205050.088432:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205050.088439:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205050.088440:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205050.088441:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205050.088443:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205050.088446:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205050.088448:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205050.088450:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205050.088453:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205050.088457:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205050.088458:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205050.088460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205050.088464:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205050.088467:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.088468:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205050.088471:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205050.088473:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205050.281638:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205050.281644:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205050.281651:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205050.281652:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205050.281655:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205050.281682:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205050.281687:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205050.281688:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205050.281749:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205050.281754:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205050.281757:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205050.600454:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205050.600460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205050.600462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205050.600466:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205050.600469:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205050.600471:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205050.600473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205050.600475:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205050.600477:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205051.048449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205051.048454:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205051.048456:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205051.048460:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205051.048462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205051.048465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.048467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205051.048468:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.048470:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205051.112478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205051.112478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205051.112486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205051.112486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205051.112491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205051.112491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205051.112499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205051.112499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205051.112505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205051.112505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205051.112512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205051.112512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.112516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205051.112521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.112524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205051.112525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205051.112528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205051.112531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205051.304483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304487:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205051.304490:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205051.304491:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205051.304493:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304495:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205051.304496:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304497:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304499:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205051.304501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205051.304503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205051.304504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205051.304517:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205051.304518:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205051.304520:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205051.624486:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301178369, at 4301178880 00000100:00000001:3.0:1788205051.624488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205051.624494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205051.624498:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205051.624503:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9960. 00000100:00100000:3.0:1788205051.624503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205051.624509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205051.624512:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205051.624515:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205051.624518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205051.624520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205051.624619:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301178369, at 4301178880 00000100:00000010:0.0:1788205051.624626:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:1.0:1788205052.072469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205052.072476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205052.072479:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205052.072484:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205052.072486:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205052.072489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205052.072491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205052.072492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205052.072494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.136473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205052.136481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205052.136483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205052.136486:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205052.136487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788205052.136491:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205052.136491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205052.136494:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205052.136494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.136496:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205052.136497:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205052.136499:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205052.136500:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205052.136502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205052.136507:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6506 00000400:00000200:0.0:1788205052.136515:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205052.136520:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205052.136523:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205052.136525:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136529:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136531:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136534:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205052.136537:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205052.136539:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205052.136542:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205052.136544:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205052.136546:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205052.136548:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205052.136551:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136553:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136554:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136556:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205052.136558:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205052.136561:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136563:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205052.136566:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205052.136569:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205052.136572:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136574:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205052.136577:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205052.136585:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000001:1.0:1788205052.136586:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000020:00000040:0.0:1788205052.136589:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 00000100:00000001:1.0:1788205052.136589:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205052.136590:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205052.136593:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205052.136595:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136597:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205052.136599:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788205052.136601:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 02000000:00000001:0.0:1788205052.136603:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205052.136603:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205052.136605:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205052.136606:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205052.136608:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205052.136610:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205052.136611:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788205052.136616:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205052.136652:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205052.136660:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205052.136663:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205052.136666:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205052.136683:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205052.136685:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205052.136688:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205052.136690:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205052.136691:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205052.136694:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205052.136697:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6506 00000400:00000200:0.0:1788205052.136701:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205052.136704:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205052.136704:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788205052.136706:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205052.136708:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136710:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136712:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205052.136713:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205052.136714:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205052.136715:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205052.136716:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205052.136718:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205052.136719:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205052.136721:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000040:3.0:1788205052.136721:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205052.136724:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205052.136726:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205052.136729:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205052.136729:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205052.136730:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136731:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136732:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205052.136733:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788205052.136735:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205052.136736:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788205052.136737:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205052.136738:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205052.136739:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136741:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.136742:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205923968:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788205052.136743:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205052.136746:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205052.136747:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205052.136749:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.136750:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205052.136751:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205052.136752:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788205052.136754:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205052.136754:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788205052.136757:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136758:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205052.136759:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20000. 00000100:00000010:0.0:1788205052.136761:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 02000000:00000001:3.0:1788205052.136761:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205052.136763:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:3.0:1788205052.136765:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788205052.136766:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205052.136767:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136768:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788205052.136769:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000001:0.0:1788205052.136770:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205052.136771:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205052.136774:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000400:00000010:3.0:1788205052.136775:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788205052.136777:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205052.136781:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205052.136781:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205923968, portal 25 00000100:00000001:3.0:1788205052.136783:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205052.136785:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000001:0.0:1788205052.136788:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205052.136790:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205052.136790:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205052.136793:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.136797:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205052.136800:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205052.136803:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205052.136806:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205923968, offset 0 00000100:00000001:0.0:1788205052.136807:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205052.136809:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205052.136812:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205052.136813:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205052.136815:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205052.136817:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205052.136817:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205052.136820:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6506 00000400:00000200:0.0:1788205052.136822:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205052.136825:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205052.136826:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205052.136827:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788205052.136829:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136831:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205052.136832:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788205052.136833:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136835:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205052.136836:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205052.136838:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205052.136838:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1788205052.136839:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205052.136841:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788205052.136842:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788205052.136843:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205052.136845:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205052.136847:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136849:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205052.136850:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788205052.136851:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205052.136852:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136853:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205052.136855:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788205052.136856:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59074:59074:256:4294967295] 192.168.201.151@tcp LPNI seq info [59054:59054:8:4294967295] 00000100:00080000:0.0:1788205052.136857:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136859:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205052.136860:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788205052.136861:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205052.136864:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205052.136865:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136867:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205052.136869:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788205052.136869:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000008:00000001:0.0:1788205052.136871:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136873:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205052.136875:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000010:0.0:1788205052.136876:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205052.136878:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205052.136880:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:3.0:1788205052.136880:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 02000000:00000001:0.0:1788205052.136881:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136883:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205052.136884:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1788205052.136885:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788205052.136886:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205052.136889:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000800:00000200:3.0:1788205052.136889:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000100:00000001:0.0:1788205052.136892:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205052.136894:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:0.0:1788205052.136895:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788205052.136897:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000100:00000001:0.0:1788205052.136902:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1788205052.136902:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136904:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205052.136904:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000040:0.0:1788205052.136907:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205052.136910:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788205052.136913:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205052.136914:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1788205052.136916:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205052.136918:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788205052.136919:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000001:0.0:1788205052.136920:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205052.136921:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205052.136921:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788205052.136923:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205052.136925:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6506 00000100:00000001:3.0:1788205052.136925:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1788205052.136928:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:3.0:1788205052.136929:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205052.136931:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205052.136933:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205052.136936:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205052.136937:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205052.136938:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136940:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205052.136941:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:3.0:1788205052.136941:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205052.136944:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205052.136945:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205052.136947:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205052.136948:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205052.136948:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.136949:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1788205052.136950:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205052.136952:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205052.136954:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205052.136954:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136956:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205052.136958:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205052.136958:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205052.136959:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205052.136960:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301183489 00000020:00000040:0.0:1788205052.136961:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205052.136963:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000010:3.0:1788205052.136965:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4f60. 00000100:00000001:0.0:1788205052.136966:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205052.136967:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205052.136969:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205052.136970:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788205052.136971:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136972:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205052.136974:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205052.136975:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301183489 00000008:00000001:0.0:1788205052.136976:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205052.136977:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4300. 00000100:00000001:0.0:1788205052.136978:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205052.136980:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205052.136981:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1788205052.136982:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:3.0:1788205052.136983:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205052.136984:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205052.136986:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.136986:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205052.136987:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205052.136988:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788205052.136989:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205052.136990:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.136992:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788205052.136993:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788205052.136996:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205052.137000:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788205052.137001:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205052.137002:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205052.137005:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788205052.137006:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205052.137007:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000001:0.0:1788205052.137008:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205052.137008:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000040:0.0:1788205052.137011:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137011:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205052.137014:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205052.137018:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205052.137022:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6517) 00000100:00000200:3.0:1788205052.137023:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137029:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205052.137030:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137032:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137034:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205052.137038:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137045:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205052.137049:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137056:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205052.137058:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137060:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137061:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137063:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205052.137064:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205052.137066:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205052.137068:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205052.137070:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205052.137073:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205052.137077:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205923968:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205052.137081:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137082:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137083:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205923968/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137091:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205052.137093:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137095:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20000. 02000000:00000001:3.0:1788205052.137097:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205052.137098:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205052.137099:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205052.137100:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205052.137102:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788205052.137104:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205052.137105:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205052.137106:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205052.137108:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205052.137109:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137110:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137115:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205052.137116:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137117:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205052.137118:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205052.137119:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205052.137121:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137125:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205052.137126:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137127:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205052.137128:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205052.137129:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.137131:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924096:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205052.137133:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205052.137134:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205052.137135:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137136:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137138:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20000. 02000000:00000001:3.0:1788205052.137139:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205052.137141:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205052.137143:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205052.137145:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205052.137147:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924096, portal 10 00000100:00000001:3.0:1788205052.137148:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205052.137149:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205052.137153:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137157:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205052.137158:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205052.137159:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205052.137161:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205924096, offset 0 00000400:00000200:3.0:1788205052.137163:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137165:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137167:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205052.137169:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205052.137171:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205052.137172:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205052.137174:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137176:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59075:59075:256:4294967295] 192.168.201.151@tcp LPNI seq info [59055:59055:8:4294967295] 00000400:00000200:3.0:1788205052.137178:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137182:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205052.137185:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137187:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205052.137189:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205052.137191:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788205052.137194:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205052.137195:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000100:3.0:1788205052.137198:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137199:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205052.137200:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205052.137202:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205052.137204:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205052.137205:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205052.137207:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205052.137209:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137214:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137215:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137219:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137220:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205052.137222:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205052.137224:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205052.137226:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205052.137227:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301179546 00000100:00000001:3.0:1788205052.137229:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137230:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301179546 00000100:00000001:3.0:1788205052.137231:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137232:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205052.137233:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137234:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205052.137236:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205052.137240:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205052.137241:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205052.137242:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205052.137243:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205052.137244:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205052.137246:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205052.137247:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205052.137252:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137259:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205052.137260:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137261:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137262:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205052.137264:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137267:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205052.137269:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137273:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205052.137274:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137275:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137276:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137276:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205052.137278:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205052.137280:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205052.137282:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205052.137283:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205052.137285:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205052.137289:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924096:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205052.137291:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205052.137292:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137296:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205052.137296:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137298:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205052.137299:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205052.137300:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.137302:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205052.137304:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205052.137305:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205052.137305:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137306:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137308:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23400. 02000000:00000001:3.0:1788205052.137309:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205052.137310:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205052.137312:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205052.137314:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205052.137315:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924224, portal 4 00000100:00000001:3.0:1788205052.137316:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205052.137317:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205052.137320:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137324:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205052.137325:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205052.137327:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205052.137328:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205924224, offset 0 00000400:00000200:3.0:1788205052.137330:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137332:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137333:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205052.137335:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205052.137336:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205052.137337:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205052.137338:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137341:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59076:59076:256:4294967295] 192.168.201.151@tcp LPNI seq info [59056:59056:8:4294967295] 00000400:00000200:3.0:1788205052.137343:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137346:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205052.137349:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137351:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205052.137353:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205052.137354:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788205052.137357:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205052.137358:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000100:3.0:1788205052.137361:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137361:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205052.137378:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205052.137380:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205052.137382:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205052.137383:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205052.137385:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205052.137387:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137391:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137393:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137396:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137397:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205052.137398:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205052.137400:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205052.137402:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205052.137403:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301179433 00000100:00000001:3.0:1788205052.137404:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137405:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301179433 00000100:00000001:3.0:1788205052.137407:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137408:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205052.137409:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137410:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205052.137412:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205052.137417:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205052.137418:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205052.137419:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205052.137420:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205052.137421:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205052.137422:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205052.137423:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205052.137428:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137431:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205052.137432:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137433:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137433:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205052.137435:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137439:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205052.137441:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137447:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205052.137449:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137450:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137452:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137453:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205052.137455:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205052.137457:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205052.137460:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205052.137462:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205052.137466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205052.137473:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205052.137477:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205052.137480:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137485:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205052.137487:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137489:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205052.137491:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205052.137492:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.137495:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205052.137498:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205052.137500:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205052.137501:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205052.137503:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137506:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21800. 02000000:00000001:3.0:1788205052.137508:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205052.137511:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205052.137513:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205052.137516:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205052.137518:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924352, portal 4 00000100:00000001:3.0:1788205052.137519:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205052.137520:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205052.137526:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137531:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205052.137533:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205052.137535:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205052.137537:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205924352, offset 0 00000400:00000200:3.0:1788205052.137542:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137545:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137548:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205052.137551:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205052.137553:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205052.137555:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205052.137557:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205052.137560:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59077:59077:256:4294967295] 192.168.201.151@tcp LPNI seq info [59057:59057:8:4294967295] 00000400:00000200:3.0:1788205052.137564:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205052.137568:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205052.137571:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137573:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205052.137575:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205052.137577:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788205052.137579:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205052.137581:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 52 secs later) 00000800:00000100:3.0:1788205052.137583:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205052.137584:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205052.137586:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205052.137588:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205052.137589:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205052.137590:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205052.137592:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205052.137594:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137598:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137600:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137604:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137605:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205052.137607:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205052.137608:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205052.137610:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205052.137612:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301179439 00000100:00000001:3.0:1788205052.137613:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137615:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301179439 00000100:00000001:3.0:1788205052.137616:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301183489 00000100:00000001:3.0:1788205052.137617:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205052.137618:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137619:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205052.137621:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205052.137641:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205052.137642:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205052.137643:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205052.137645:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205052.137646:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205052.137647:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205052.137649:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205052.137654:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137658:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205052.137659:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137660:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137660:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205052.137662:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137665:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205052.137668:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137671:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205052.137673:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137674:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205052.137675:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137676:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205052.137677:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205052.137679:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205052.137680:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205052.137681:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205052.137683:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205052.137688:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924352:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205052.137690:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137691:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137693:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205924096/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137696:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205052.137697:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137698:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20000. 02000000:00000001:3.0:1788205052.137699:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205052.137700:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205052.137701:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205052.137702:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205052.137703:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:3.0:1788205052.137704:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205052.137705:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205052.137706:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205052.137707:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205052.137708:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137709:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137710:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205924224/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137718:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205052.137718:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137719:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23400. 02000000:00000001:3.0:1788205052.137721:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205052.137721:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205052.137722:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205052.137723:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205052.137724:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788205052.137725:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205052.137725:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205052.137726:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205052.137727:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205052.137728:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137728:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205052.137730:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205924352/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205052.137733:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205052.137734:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205052.137734:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21800. 02000000:00000001:3.0:1788205052.137735:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205052.137736:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205052.137737:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205052.137737:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205052.137738:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788205052.137740:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205052.137740:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205052.137741:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205052.137742:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205052.137743:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137744:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.137746:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205052.137746:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205052.137747:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205052.137748:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205052.328574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205052.328580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205052.328586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205052.328588:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205052.328590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205052.328591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205052.328595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205052.328803:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205052.330067:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205052.330071:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205052.330074:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205052.648474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205052.648481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205052.648484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205052.648490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205052.648493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205052.648497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.648500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205052.648503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205052.648505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.096427:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205053.096435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205053.096438:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.096444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205053.096448:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205053.096452:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.096454:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205053.096456:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.096459:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.160506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205053.160506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205053.160515:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205053.160515:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205053.160519:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.160520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.160527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205053.160527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205053.160533:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205053.160533:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205053.160539:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.160539:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.160543:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.160547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.160551:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.160551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.160555:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.160557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205053.352543:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205053.352545:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205053.352548:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352551:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205053.352553:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352554:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352555:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205053.352558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205053.352562:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205053.352563:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205053.352564:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352565:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205053.352566:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352567:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352568:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205053.352571:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205053.352572:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205053.352573:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352574:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205053.352575:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352576:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205053.352577:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205053.352591:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205053.352593:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205053.352596:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205053.672553:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205053.672562:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205053.672573:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205053.672579:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205053.672583:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205053.672587:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205053.672589:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205053.672591:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205053.672595:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205054.120579:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205054.120586:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205054.120588:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205054.120622:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205054.120626:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205054.120629:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.120632:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205054.120634:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.120637:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205054.184947:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205054.184947:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205054.184954:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205054.184955:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205054.184956:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205054.184958:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205054.184962:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205054.184964:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205054.184968:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205054.184969:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205054.184972:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205054.184973:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.184975:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205054.184978:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.184980:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205054.184984:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205054.184987:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205054.184990:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205054.376506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205054.376512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205054.376519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205054.376520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205054.376523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205054.376524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205054.376529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205054.376531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205054.376605:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205054.376609:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205054.376613:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205054.696494:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205054.696502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205054.696504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205054.696509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205054.696512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205054.696516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205054.696519:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205054.696522:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205054.696525:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205055.144476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205055.144482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205055.144484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205055.144490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205055.144493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205055.144497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205055.144500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205055.144502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205055.144505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205055.209040:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205055.209047:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205055.209049:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205055.209054:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205055.209058:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205055.209061:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205055.209064:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205055.209067:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205055.209070:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205055.209236:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205055.209239:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205055.209241:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205055.209244:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205055.209246:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205055.209248:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205055.209249:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205055.209251:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205055.209253:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205055.400535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205055.400541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205055.400547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205055.400548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205055.400551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205055.400552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205055.400556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205055.400557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205055.400604:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205055.400607:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205055.400610:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205055.720430:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205055.720437:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205055.720439:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205055.720444:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205055.720448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205055.720452:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205055.720455:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205055.720458:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205055.720461:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205056.168526:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205056.168755:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205056.168756:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205056.168762:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205056.168765:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205056.168769:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205056.168771:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205056.168774:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205056.168777:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205056.232501:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205056.232501:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205056.232512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205056.232512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205056.232517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205056.232517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205056.232526:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205056.232527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205056.232532:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205056.232533:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205056.232539:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205056.232540:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205056.232544:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205056.232549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205056.232552:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205056.232553:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205056.232565:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205056.232568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205056.424501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205056.424509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205056.424516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205056.424518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205056.424522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205056.424523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205056.424527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205056.424528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205056.424601:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205056.424607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205056.424612:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205056.744501:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205056.744507:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205056.744509:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205056.744514:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205056.744517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205056.744521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205056.744523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205056.744526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205056.744529:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205056.744609:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301183489, at 4301184000 00000100:00000040:1.0:1788205056.744620:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301183489, at 4301184000 00000100:00000010:0.0:1788205056.744627:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000010:1.0:1788205056.744629:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000001:1.0:1788205057.195765:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205057.195770:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205057.195772:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205057.195775:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205057.195778:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205057.195781:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205057.195782:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205057.195784:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205057.195786:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.256469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205057.256476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205057.256477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205057.256481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205057.256484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205057.256487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205057.256489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205057.256491:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205057.256493:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205057.257099:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205057.257103:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205057.257104:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205057.257107:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205057.257109:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205057.257111:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205057.257113:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205057.257116:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205057.257118:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205057.258888:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205057.258895:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205057.258897:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205057.258900:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205057.258904:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205057.258912:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6512 00000400:00000200:3.0:1788205057.258920:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205057.258926:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205057.258929:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205057.258932:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.258934:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.258937:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.258939:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205057.258942:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205057.258944:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205057.258947:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205057.258950:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205057.258952:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205057.258955:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205057.258958:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.258959:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.258961:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.258962:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205057.258965:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205057.258968:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205057.258970:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205057.258973:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205057.258976:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205057.258979:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.258981:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205057.258983:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205057.258992:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2a00. 00000020:00000040:3.0:1788205057.258996:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205057.258999:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205057.259001:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259003:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205057.259006:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205057.259010:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205057.259017:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21800. 00000100:00000001:3.0:1788205057.259024:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205057.259033:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205057.259042:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205057.259047:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205057.259051:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c42bbc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205057.259075:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205057.259079:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205057.259082:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205057.259084:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205057.259086:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205057.259089:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205057.259092:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6512 00000400:00000200:3.0:1788205057.259096:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205057.259100:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205057.259102:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205057.259106:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259108:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259110:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259112:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205057.259114:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205057.259118:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1788205057.259118:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205057.259122:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788205057.259127:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205057.259128:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:0.0:1788205057.259133:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788205057.259134:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205057.259139:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205057.259148:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205057.259149:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205057.259156:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259165:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259174:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205057.259178:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1788205057.259180:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:0.0:1788205057.259182:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205057.259188:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:0.0:1788205057.259190:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205057.259192:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.259196:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205057.259197:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205057.259205:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205057.259205:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205057.259212:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205057.259216:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205924480:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205057.259216:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205057.259225:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788205057.259228:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1788205057.259231:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.259235:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205057.259236:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.259242:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205057.259244:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1f80. 00000020:00000040:3.0:1788205057.259249:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205057.259252:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205057.259253:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259255:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205057.259257:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205057.259259:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205057.259259:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205057.259268:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23400. 02000000:00000010:0.0:1788205057.259272:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:3.0:1788205057.259273:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205057.259283:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205057.259291:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205057.259293:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205057.259297:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002399dd83] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205057.259306:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205057.259309:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205057.259312:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205057.259313:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205057.259315:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205057.259317:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205057.259320:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6512 00000400:00000200:3.0:1788205057.259323:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205057.259326:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205057.259329:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205057.259331:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259333:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259334:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259336:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205057.259338:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205057.259339:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205057.259343:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788205057.259344:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205057.259346:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788205057.259353:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205057.259356:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205057.259387:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205057.259395:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000100:00000040:3.0:1788205057.259402:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:0.0:1788205057.259406:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000001:3.0:1788205057.259407:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259417:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259420:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:0.0:1788205057.259422:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924480, portal 25 00000020:00000040:3.0:1788205057.259423:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000001:0.0:1788205057.259428:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788205057.259431:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205057.259433:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000001:3.0:1788205057.259437:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205057.259441:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205057.259445:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205057.259447:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259449:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205057.259451:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205057.259453:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259455:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205057.259459:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1180. 00000100:00000040:0.0:1788205057.259462:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205057.259464:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205057.259475:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205057.259477:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259479:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205057.259483:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205057.259487:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.259494:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:3.0:1788205057.259496:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20000. 00000100:00000040:0.0:1788205057.259503:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259503:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205057.259511:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00100000:3.0:1788205057.259517:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1788205057.259518:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205924480, offset 0 00000400:00000200:0.0:1788205057.259533:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259534:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205057.259550:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205057.259556:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003da94af2] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205057.259564:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205057.259568:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205057.259572:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205057.259573:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205057.259575:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205057.259577:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205057.259579:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6512 00000400:00000200:3.0:1788205057.259584:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205057.259587:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205057.259589:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205057.259592:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259594:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259596:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259598:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205057.259600:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205057.259601:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205057.259603:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205057.259605:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205057.259606:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205057.259608:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205057.259610:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259612:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205057.259613:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259614:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205057.259616:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205057.259618:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205057.259620:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205057.259624:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205057.259627:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205057.259629:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259631:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205057.259633:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:0.0:1788205057.259638:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:3.0:1788205057.259639:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.259645:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205057.259650:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1880. 00000400:00000200:0.0:1788205057.259656:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:3.0:1788205057.259656:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205057.259666:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205057.259669:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.259671:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205057.259676:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205057.259679:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205057.259682:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.259683:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000010:3.0:1788205057.259692:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20800. 00000400:00000200:0.0:1788205057.259694:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205057.259700:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.259702:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205057.259716:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59078:59078:256:4294967295] 192.168.201.151@tcp LPNI seq info [59058:59058:8:4294967295] 00000100:00100000:3.0:1788205057.259716:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205057.259726:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205057.259728:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205057.259732:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072ac1564] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205057.259738:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205057.259743:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6522) 00000400:00000200:0.0:1788205057.259787:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.259799:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205057.259806:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.259814:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205057.259822:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205057.259827:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788205057.259832:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205057.259836:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788205057.259841:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.259844:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205057.259854:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205057.259859:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205057.259865:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205057.259869:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205057.259874:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205057.259879:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.259891:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.259895:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.259903:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.259905:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205057.259913:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.259917:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205057.259920:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301188612 00000100:00000010:0.0:1788205057.259926:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205057.259934:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205057.259943:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301188612 00000100:00000010:0.0:1788205057.259946:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a80. 00000100:00000001:0.0:1788205057.259949:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205057.259953:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205057.259956:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.259959:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205057.259964:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205057] req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205057.259975:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205057.259977:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205057.259980:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205057.259982:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205057.259984:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205057.259987:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205057.259991:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205057.260001:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260008:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205057.260009:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260012:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260014:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205057.260019:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260025:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205057.260030:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260037:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205057.260040:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260042:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260043:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260047:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205057.260049:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205057.260052:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205057.260056:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205057.260059:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205057.260063:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205057.260071:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205924480:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205057.260077:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260079:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.260083:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205924480/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260095:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205057.260098:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205057.260101:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205057.260106:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205057.260109:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205057.260111:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205057.260113:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205057.260116:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21800. 02000000:00000001:0.0:1788205057.260121:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205057.260123:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205057.260125:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:0.0:1788205057.260130:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205057.260131:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260134:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260143:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205057.260145:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.260148:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205057.260149:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205057.260151:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205057.260155:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260161:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205057.260162:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260165:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205057.260167:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205057.260170:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205057.260173:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924608:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205057.260176:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205057.260179:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205057.260181:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260183:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205057.260186:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205057.260188:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.260193:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205057.260196:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205057.260200:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205057.260204:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924608, portal 10 00000100:00000001:0.0:1788205057.260206:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205057.260207:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:0.0:1788205057.260216:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260222:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205057.260224:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205057.260226:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205057.260228:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205924608, offset 0 00000400:00000200:0.0:1788205057.260232:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205057.260235:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.260240:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205057.260243:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205057.260245:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205057.260247:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205057.260249:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205057.260252:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59079:59079:256:4294967295] 192.168.201.151@tcp LPNI seq info [59059:59059:8:4294967295] 00000400:00000200:0.0:1788205057.260256:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.260262:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205057.260266:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.260270:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205057.260273:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205057.260332:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788205057.260338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205057.260341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788205057.260344:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.260346:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205057.260350:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205057.260354:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205057.260357:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205057.260359:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205057.260361:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205057.260386:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260393:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.260396:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260403:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.260405:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205057.260408:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205057.260412:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205057.260415:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205057.260416:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301184670 00000100:00000001:0.0:1788205057.260419:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.260421:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301184670 00000100:00000001:0.0:1788205057.260423:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.260425:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205057.260427:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.260429:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205057.260432:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205057] req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205057.260439:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205057.260441:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205057.260442:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205057.260445:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205057.260446:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205057.260448:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205057.260451:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205057.260459:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260469:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205057.260470:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260472:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260473:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205057.260477:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260483:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205057.260487:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260493:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205057.260495:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260496:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260497:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260498:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205057.260501:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205057.260503:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205057.260507:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205057.260509:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205057.260513:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205057.260520:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924608:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205057.260523:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205057.260527:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260534:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205057.260552:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260555:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205057.260557:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205057.260611:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205057.260615:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924736:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205057.260618:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205057.260620:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205057.260621:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260623:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205057.260627:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205057.260629:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.260632:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205057.260635:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205057.260638:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205057.260640:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924736, portal 4 00000100:00000001:0.0:1788205057.260642:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205057.260644:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:0.0:1788205057.260650:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260656:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205057.260659:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205057.260661:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205057.260663:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205924736, offset 0 00000400:00000200:0.0:1788205057.260666:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205057.260670:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.260673:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205057.260676:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205057.260679:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205057.260681:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205057.260683:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205057.260686:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59080:59080:256:4294967295] 192.168.201.151@tcp LPNI seq info [59060:59060:8:4294967295] 00000400:00000200:0.0:1788205057.260689:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.260695:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205057.260700:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.260703:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205057.260707:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205057.260709:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788205057.260713:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205057.260769:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788205057.260774:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.260777:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205057.260780:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205057.260784:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205057.260787:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205057.260789:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205057.260791:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205057.260794:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260802:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.260805:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260811:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.260813:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205057.260816:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205057.260818:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205057.260822:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205057.260823:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301184556 00000100:00000001:0.0:1788205057.260826:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.260828:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301184556 00000100:00000001:0.0:1788205057.260830:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.260832:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205057.260835:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.260836:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205057.260840:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205057] req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205057.260848:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205057.260849:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205057.260851:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205057.260853:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205057.260855:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205057.260857:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205057.260860:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205057.260869:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260875:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205057.260877:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260879:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260880:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205057.260884:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260890:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205057.260893:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260900:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205057.260902:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260903:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.260905:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.260906:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205057.260909:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205057.260911:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205057.260914:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205057.260917:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205057.260921:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205057.260928:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924736:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205057.260933:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205057.260937:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260942:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205057.260944:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260946:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205057.260948:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205057.260950:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205057.260953:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924864:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205057.260956:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205057.260958:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205057.260960:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205057.260962:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205057.260965:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205057.260967:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205057.260970:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205057.260972:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205057.260975:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205057.260978:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924864, portal 4 00000100:00000001:0.0:1788205057.260980:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205057.260981:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:0.0:1788205057.260986:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.260992:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205057.260994:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205057.260996:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205057.260999:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205924864, offset 0 00000400:00000200:0.0:1788205057.261003:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205057.261006:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.261009:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205057.261012:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205057.261015:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205057.261018:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205057.261020:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205057.261023:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59081:59081:256:4294967295] 192.168.201.151@tcp LPNI seq info [59061:59061:8:4294967295] 00000400:00000200:0.0:1788205057.261027:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205057.261033:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205057.261037:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.261040:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205057.261043:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205057.261046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788205057.261050:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205057.261052:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788205057.261072:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205057.261074:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205057.261078:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205057.261081:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205057.261084:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205057.261085:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205057.261088:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205057.261092:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261099:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.261103:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261109:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.261110:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205057.261113:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205057.261116:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205057.261119:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205057.261122:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301184562 00000100:00000001:0.0:1788205057.261124:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.261126:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301184562 00000100:00000001:0.0:1788205057.261128:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301188612 00000100:00000001:0.0:1788205057.261130:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205057.261132:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.261134:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205057.261137:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205057] req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205057.261144:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205057.261146:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205057.261147:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205057.261149:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205057.261151:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205057.261153:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205057.261157:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205057.261165:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261171:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205057.261172:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261173:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.261174:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205057.261177:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261184:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205057.261187:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261192:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205057.261195:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261197:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205057.261198:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261200:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205057.261202:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205057.261204:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205057.261206:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205057.261208:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205057.261211:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205057.261219:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205924864:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205057.261223:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261225:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.261228:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205924608/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261235:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205057.261236:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205057.261238:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205057.261240:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205057.261242:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205057.261244:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205057.261246:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205057.261249:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23400. 02000000:00000001:0.0:1788205057.261251:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205057.261252:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205057.261254:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:0.0:1788205057.261257:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205057.261258:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261260:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.261262:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205924736/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261274:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205057.261331:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205057.261334:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205057.261337:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205057.261339:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205057.261340:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205057.261342:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205057.261344:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20000. 02000000:00000001:0.0:1788205057.261346:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205057.261348:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205057.261349:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:0.0:1788205057.261351:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205057.261352:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261354:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205057.261358:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205924864/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205057.261383:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205057.261384:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205057.261386:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205057.261388:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205057.261389:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205057.261391:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205057.261393:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205057.261394:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20800. 02000000:00000001:0.0:1788205057.261395:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205057.261397:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205057.261398:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:0.0:1788205057.261400:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205057.261401:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261403:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205057.261408:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205057.261409:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205057.261411:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205057.261413:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205057.453656:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205057.453662:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205057.453668:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205057.453669:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205057.453672:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205057.453673:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205057.453677:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205057.453678:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205057.453765:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205057.453773:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205057.453779:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205057.769635:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205057.769642:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205057.769644:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205057.769649:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205057.769652:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205057.769656:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205057.769659:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205057.769661:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205057.769664:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.217026:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205058.217035:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205058.217037:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.217041:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205058.217044:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205058.217048:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.217051:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.217053:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.217055:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.280855:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205058.280863:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205058.280864:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205058.280869:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205058.280872:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205058.280876:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205058.280879:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205058.280882:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205058.280884:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.283441:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205058.283447:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205058.283449:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.283453:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205058.283455:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205058.283458:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.283460:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.283462:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.283465:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205058.473546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205058.473553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205058.473558:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205058.473560:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205058.473563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205058.473564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205058.473567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205058.473569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205058.473689:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205058.473692:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205058.473695:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205058.793982:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205058.793988:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205058.793990:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205058.793994:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205058.793997:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205058.794001:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205058.794003:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205058.794006:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205058.794009:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205059.241051:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205059.241058:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205059.241061:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205059.241065:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205059.241068:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205059.241071:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205059.241073:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205059.241076:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205059.241078:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205059.304463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205059.304471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205059.304473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205059.304479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205059.304482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205059.304486:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205059.304489:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205059.304491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205059.304494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.304610:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205059.304613:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205059.304615:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205059.304617:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205059.304619:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205059.304621:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205059.304623:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205059.305106:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205059.305110:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205059.497793:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205059.497799:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205059.497804:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205059.497805:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205059.497809:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205059.497810:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205059.497813:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205059.497814:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205059.498649:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205059.498652:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205059.498655:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205059.816469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205059.816475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205059.816477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205059.816481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205059.816483:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205059.816487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205059.816489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205059.816499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205059.816503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205060.264666:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205060.264673:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205060.264674:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205060.264679:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205060.264682:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205060.264686:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205060.264698:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205060.264701:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205060.264704:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205060.328505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205060.328512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205060.328514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205060.328519:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205060.328523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205060.328527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205060.328530:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205060.328532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205060.328535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205060.329984:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205060.329989:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205060.329991:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205060.329994:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205060.329996:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205060.329998:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205060.330000:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205060.330002:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205060.330004:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205060.520510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205060.520518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205060.520523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205060.520524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205060.520527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205060.520528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205060.520531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205060.520532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205060.520596:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205060.520600:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205060.520603:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205060.840913:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205060.840921:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205060.840923:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205060.840928:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205060.840931:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205060.840934:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205060.840937:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205060.840940:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205060.840942:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205061.288451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205061.288458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205061.288461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205061.288499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205061.288504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205061.288507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205061.288510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205061.288513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205061.288516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205061.352487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205061.352496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205061.352498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205061.352500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205061.352502:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205061.352504:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205061.352510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205061.352510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205061.352516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205061.352517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205061.352520:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205061.352521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205061.352524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205061.352527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205061.352530:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205061.352532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205061.352535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205061.352537:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545735:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205061.545742:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545748:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205061.545751:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205061.545754:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205061.545758:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545761:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205061.545763:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545765:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545767:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545771:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205061.545773:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205061.545775:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205061.545777:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205061.545973:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205061.545976:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205061.545979:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205061.864572:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205061.864577:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205061.864579:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205061.864583:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205061.864585:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205061.864589:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205061.864591:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205061.864593:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205061.864594:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205061.865613:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301188612, at 4301189121 00000100:00000010:1.0:1788205061.865623:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a80. 00000100:00000040:0.0:1788205061.865708:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301188612, at 4301189121 00000100:00000010:0.0:1788205061.865714:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:1.0:1788205062.312503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205062.312509:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205062.312511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205062.312516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205062.312520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205062.312523:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205062.312526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205062.312528:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205062.312531:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205062.376550:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205062.376559:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205062.376561:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205062.376566:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205062.376570:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205062.376574:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205062.376576:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205062.376579:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205062.376583:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205062.376584:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205062.376591:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205062.376594:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205062.376597:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205062.376602:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205062.376610:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6517 00000400:00000200:2.0:1788205062.376619:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205062.376625:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205062.376627:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205062.376630:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376633:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376635:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376637:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205062.376641:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205062.376644:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205062.376646:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205062.376650:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205062.376653:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205062.376656:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205062.376660:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376662:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376664:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376667:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205062.376669:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205062.376672:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376675:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205062.376678:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205062.376681:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205062.376684:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205062.376685:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205062.376686:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205062.376690:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205062.376691:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205062.376694:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205062.376698:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205062.376701:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1788205062.376702:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000100:00000001:1.0:1788205062.376704:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788205062.376706:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 00000100:00000001:1.0:1788205062.376708:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205062.376712:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205062.376714:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205062.376715:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205062.376717:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376722:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205062.376724:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205062.376729:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205062.376737:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205062.376744:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.376753:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205062.376764:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205062.376767:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205062.376772:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205062.376800:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205062.376804:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205062.376807:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205062.376809:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205062.376810:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205062.376813:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205062.376815:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6517 00000400:00000200:2.0:1788205062.376819:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205062.376821:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205062.376824:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205062.376826:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376828:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376830:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376831:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205062.376833:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205062.376834:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205062.376836:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205062.376838:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205062.376840:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205062.376841:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205062.376844:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376846:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376847:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376849:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205062.376851:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205062.376853:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376855:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205062.376856:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205062.376861:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205062.376863:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376864:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205062.376868:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205062.376871:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376873:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205062.376877:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205062.376879:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205062.376881:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205062.376883:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376885:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205062.376886:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205062.376888:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205062.376891:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788205062.376894:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.376899:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205062.376906:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205062.376908:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205062.376912:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205062.376919:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205062.376921:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205062.376925:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205062.376926:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205062.376928:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205062.376931:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205062.376933:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6517 00000400:00000200:2.0:1788205062.376936:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205062.376939:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205062.376941:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205062.376944:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376946:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376948:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376950:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205062.376952:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205062.376954:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205062.376956:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205062.376958:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205062.376959:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205062.376962:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205062.376964:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376966:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.376968:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376970:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205062.376972:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205062.376974:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205062.376977:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205062.376978:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205062.376981:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205062.376983:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376984:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205062.376986:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205062.376987:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.376989:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205062.376993:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205062.376995:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205062.376997:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205062.376998:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.377000:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205062.377001:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205062.377002:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205062.377005:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205062.377008:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.377013:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205062.377019:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205062.377021:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205062.377026:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205062.377034:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205062.377036:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205062.377039:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205062.377041:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205062.377042:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205062.377044:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205062.377046:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6517 00000400:00000200:2.0:1788205062.377050:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205062.377053:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205062.377055:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205062.377057:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.377059:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.377061:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.377062:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205062.377064:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205062.377066:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205062.377067:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205062.377069:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205062.377071:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205062.377072:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205062.377075:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.377076:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205062.377078:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205062.377079:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205062.377081:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205062.377083:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205062.377085:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205062.377087:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205062.377089:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205062.377091:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.377092:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205062.377094:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205062.377096:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.377098:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205062.377103:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205062.377105:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205062.377107:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205062.377108:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205062.377110:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205062.377111:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205062.377112:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205062.377116:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205062.377119:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.377123:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205062.377129:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205062.377131:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205062.377134:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205062.377142:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205062.377146:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6527) 00000100:00000001:0.0:1788205062.377217:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205062.377220:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205062.377222:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205062.377228:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377236:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205062.377238:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377241:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205062.377244:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205062.377247:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205062.377250:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205924992:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205062.377254:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205062.377257:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205062.377260:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377262:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205062.377266:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205062.377268:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205062.377272:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205062.377277:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205062.377283:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205062.377289:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205924992, portal 25 00000100:00000001:0.0:1788205062.377291:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205062.377293:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788205062.377298:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377304:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205062.377307:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205062.377310:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205062.377312:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205924992, offset 0 00000400:00000200:0.0:1788205062.377318:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205062.377326:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.377332:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205062.377338:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205062.377342:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205062.377345:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205062.377348:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205062.377351:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59082:59082:256:4294967295] 192.168.201.151@tcp LPNI seq info [59062:59062:8:4294967295] 00000400:00000200:0.0:1788205062.377355:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.377388:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205062.377395:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.377400:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205062.377407:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205062.377410:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788205062.377416:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205062.377419:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788205062.377423:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.377502:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205062.377511:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205062.377516:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205062.377521:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205062.377524:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205062.377528:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205062.377532:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377542:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.377545:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377552:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.377554:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205062.377559:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205062.377563:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205062.377566:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301193730 00000100:00000010:0.0:1788205062.377570:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205062.377576:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205062.377581:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301193730 00000100:00000010:0.0:1788205062.377584:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff91e0. 00000100:00000001:0.0:1788205062.377587:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205062.377591:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205062.377593:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.377596:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205062.377600:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205062/real 1788205062] req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205062.377608:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205062.377610:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205062.377612:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205062.377614:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205062.377616:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205062.377618:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205062.377621:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205062.377629:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377641:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205062.377643:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377645:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.377647:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205062.377651:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377657:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205062.377661:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377667:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205062.377669:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377670:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.377672:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377674:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205062.377675:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205062.377677:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205062.377681:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205062.377683:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205062.377687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205062.377694:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205924992:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205062.377698:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205062.377702:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377707:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205062.377708:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377710:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205062.377713:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205062.377715:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205062.377717:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925120:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205062.377720:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205062.377722:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205062.377723:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377725:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205062.377728:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205062.377730:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205062.377733:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205062.377736:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205062.377739:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205062.377741:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925120, portal 10 00000100:00000001:0.0:1788205062.377743:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205062.377744:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788205062.377750:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377757:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205062.377759:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205062.377761:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205062.377763:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205925120, offset 0 00000400:00000200:0.0:1788205062.377767:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205062.377771:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.377774:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205062.377777:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205062.377779:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205062.377781:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205062.377784:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205062.377787:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59083:59083:256:4294967295] 192.168.201.151@tcp LPNI seq info [59063:59063:8:4294967295] 00000400:00000200:0.0:1788205062.377790:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.377796:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205062.377800:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.377804:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205062.377807:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205062.377810:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788205062.377814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205062.377817:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788205062.377820:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.377822:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205062.377825:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205062.377829:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205062.377832:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205062.377833:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205062.377836:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205062.377839:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377847:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.377850:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377856:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.377857:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205062.377860:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205062.377864:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205062.377867:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205062.377869:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301189787 00000100:00000001:0.0:1788205062.377872:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.377873:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301189787 00000100:00000001:0.0:1788205062.377875:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.377877:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205062.377879:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.377881:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205062.377884:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205062/real 1788205062] req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205062.377898:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205062.377899:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205062.377901:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205062.377904:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205062.377905:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205062.377907:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205062.377910:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205062.377917:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377923:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205062.377924:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377926:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.377927:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205062.377930:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377935:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205062.377938:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377944:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205062.377946:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377947:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.377948:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.377950:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205062.377951:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205062.377954:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205062.377957:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205062.377959:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205062.377962:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205062.377969:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925120:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205062.377972:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205062.377975:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.377980:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205062.377981:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377983:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205062.377985:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205062.377986:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205062.377989:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205062.377991:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205062.377993:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205062.377994:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.377995:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205062.377999:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205062.378001:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205062.378003:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205062.378006:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205062.378008:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205062.378011:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925248, portal 4 00000100:00000001:0.0:1788205062.378012:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205062.378013:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788205062.378019:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378024:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205062.378026:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205062.378028:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205062.378031:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205925248, offset 0 00000400:00000200:0.0:1788205062.378035:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205062.378038:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.378041:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205062.378044:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205062.378046:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205062.378049:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205062.378051:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205062.378053:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59084:59084:256:4294967295] 192.168.201.151@tcp LPNI seq info [59064:59064:8:4294967295] 00000400:00000200:0.0:1788205062.378057:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.378063:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205062.378068:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.378071:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205062.378075:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205062.378077:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788205062.378081:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205062.378083:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788205062.378087:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.378089:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205062.378092:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205062.378096:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205062.378098:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205062.378100:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205062.378102:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205062.378105:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378113:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.378116:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378122:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.378123:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205062.378126:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205062.378128:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205062.378131:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205062.378134:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301189673 00000100:00000001:0.0:1788205062.378136:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.378138:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301189673 00000100:00000001:0.0:1788205062.378139:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.378141:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205062.378143:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.378145:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205062.378148:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205062/real 1788205062] req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205062.378154:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205062.378156:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205062.378157:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205062.378159:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205062.378161:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205062.378163:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205062.378165:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205062.378173:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378178:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205062.378180:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.378181:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.378183:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205062.378186:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378193:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205062.378196:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378202:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205062.378205:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.378207:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.378208:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.378210:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205062.378212:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205062.378214:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205062.378217:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205062.378219:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205062.378223:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205062.378230:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205062.378234:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205062.378237:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378243:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205062.378244:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.378246:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205062.378247:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205062.378250:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205062.378252:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925376:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205062.378255:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205062.378257:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205062.378259:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205062.378260:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205062.378263:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788205062.378265:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205062.378268:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205062.378271:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205062.378273:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205062.378276:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925376, portal 4 00000100:00000001:0.0:1788205062.378278:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205062.378279:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205062.378284:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378290:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205062.378292:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205062.378294:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205062.378296:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205925376, offset 0 00000400:00000200:0.0:1788205062.378300:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205062.378303:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.378306:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205062.378308:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205062.378310:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205062.378312:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205062.378314:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205062.378316:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59085:59085:256:4294967295] 192.168.201.151@tcp LPNI seq info [59065:59065:8:4294967295] 00000400:00000200:0.0:1788205062.378320:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205062.378326:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205062.378330:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.378333:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205062.378337:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205062.378339:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788205062.378343:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205062.378345:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788205062.378349:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205062.378350:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205062.378353:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205062.378356:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205062.378359:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205062.378360:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205062.378384:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205062.378388:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378400:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.378403:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205087 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.378409:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.378410:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205062.378413:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205062.378416:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205062.378419:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205062.378420:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301189680 00000100:00000001:0.0:1788205062.378422:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.378424:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301189680 00000100:00000001:0.0:1788205062.379535:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301193730 00000100:00000001:0.0:1788205062.379540:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205062.379544:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.379546:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205062.379553:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205062/real 1788205062] req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205062.379563:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205062.379565:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205062.379567:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205062.379571:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205062.379572:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205062.379575:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205062.379579:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205062.379586:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379593:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205062.379595:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379597:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.379599:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205062.379603:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379608:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205062.379612:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379618:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205062.379621:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379623:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205062.379624:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379626:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205062.379628:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205062.379630:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205062.379634:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205062.379637:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205062.379640:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205062.379649:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925376:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205062.379653:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379655:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.379660:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205924992/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379667:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205062.379670:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205062.379672:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205062.379675:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205062.379678:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205062.379680:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205062.379682:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205062.379685:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205062.379689:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205062.379691:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205062.379692:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205062.379696:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205062.379698:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379699:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.379703:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205925120/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379709:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205062.379711:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205062.379712:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205062.379715:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205062.379716:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205062.379718:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205062.379720:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205062.379721:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205062.379724:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205062.379725:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205062.379726:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205062.379728:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205062.379728:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379730:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.379733:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205925248/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379739:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205062.379741:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205062.379742:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205062.379744:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205062.379746:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205062.379747:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205062.379749:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205062.379750:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205062.379752:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205062.379754:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205062.379755:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205062.379757:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205062.379758:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379759:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205062.379762:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205925376/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205087 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205062.379768:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205062.379769:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205062.379770:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788205062.379772:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205062.379773:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205062.379774:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205062.379776:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205062.379777:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:0.0:1788205062.379779:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205062.379780:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205062.379781:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205062.379784:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205062.379785:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379786:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205062.379793:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205062.379794:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205062.379797:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205062.379799:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205062.568495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205062.568502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205062.568508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205062.568510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205062.568513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205062.568514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205062.568517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205062.568519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205062.568580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205062.568584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205062.568587:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205062.888588:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205062.888595:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205062.888598:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.888602:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205062.888605:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205062.888608:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205062.888611:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205062.888613:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205062.888616:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205063.338211:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205063.338219:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205063.338221:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205063.338227:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205063.338230:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205063.338233:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205063.338234:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205063.338236:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205063.338238:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205063.400490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205063.400496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205063.400497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205063.400501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205063.400504:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205063.400507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205063.400509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205063.400511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205063.400512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205063.400581:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205063.400583:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205063.400584:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205063.400586:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205063.400587:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205063.400589:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205063.400590:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205063.400591:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205063.400592:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205063.592575:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205063.592578:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205063.592582:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592587:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205063.592589:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592590:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592592:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205063.592597:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592598:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205063.592600:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205063.592601:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205063.592603:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592605:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205063.592606:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592607:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592609:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592613:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205063.592614:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205063.592615:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205063.592617:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592618:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205063.592620:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592621:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205063.592623:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205063.592689:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205063.592693:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205063.592697:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205063.912474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205063.912480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205063.912481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205063.912484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205063.912487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205063.912490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205063.912492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205063.912494:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205063.912495:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.360502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205064.360508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205064.360510:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.360524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205064.360527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205064.360531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.360533:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.360535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.360537:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.424491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205064.424491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205064.424499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205064.424499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205064.424503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205064.424504:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.424511:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205064.424511:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205064.424514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205064.424518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205064.424520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205064.424521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.424524:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205064.424524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.424528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.424531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.424532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.424535:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205064.616501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205064.616508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205064.616521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205064.616522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205064.616525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205064.616526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205064.616529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205064.616530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205064.618131:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205064.618137:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205064.618141:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205064.936536:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205064.936546:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205064.936548:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205064.936553:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205064.936557:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205064.936560:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.936562:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205064.936564:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.936566:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205065.384485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205065.384491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205065.384493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205065.384497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205065.384499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205065.384503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205065.384506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205065.384508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205065.384511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205065.448491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205065.448491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205065.448500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205065.448500:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205065.448503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205065.448503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205065.448510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205065.448510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205065.448516:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205065.448517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205065.448521:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205065.448521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205065.448527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205065.448530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205065.448533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205065.448534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205065.448537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205065.448540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205065.640574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205065.640580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205065.640586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205065.640587:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205065.640590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205065.640592:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205065.640595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205065.640596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205065.640671:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205065.640675:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205065.640678:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205065.960490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205065.960498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205065.960501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205065.960506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205065.960512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205065.960516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.960520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205065.960523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.960526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205066.408483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205066.408489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205066.408491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205066.408496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205066.408499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205066.408502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.408504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205066.408507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.408510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205066.472532:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205066.472539:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205066.472541:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205066.472546:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205066.472550:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205066.472554:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205066.472557:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205066.472560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205066.472563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205066.475472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205066.475477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205066.475479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205066.475483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205066.475485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205066.475488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.475490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205066.475492:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.475494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205066.664562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205066.664569:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205066.664575:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205066.664577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205066.664580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205066.664581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205066.664584:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205066.664585:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205066.664644:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205066.664647:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205066.664649:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205066.984499:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301193730, at 4301194240 00000100:00000001:2.0:1788205066.984501:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205066.984508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205066.984512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205066.984516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205066.984518:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff91e0. 00000100:00000001:2.0:1788205066.984520:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205066.984525:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205066.984528:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205066.984531:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205066.984534:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205066.985150:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301193730, at 4301194240 00000100:00000010:0.0:1788205066.985157:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:1.0:1788205067.432484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205067.432491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205067.432494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205067.432499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205067.432503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205067.432506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205067.432509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205067.432510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205067.432513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205067.496510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205067.496513:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205067.496519:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205067.496520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205067.496523:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205067.496525:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205067.496535:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205067.496540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205067.496542:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205067.496543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205067.496545:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205067.496552:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205067.496553:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205067.496554:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205067.496564:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205067.496564:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205067.496565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205067.496573:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205067.496577:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205067.496578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.496579:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205067.496583:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205067.496589:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205067.496590:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6522 00000400:00000200:2.0:1788205067.496605:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205067.496613:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205067.496617:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205067.496621:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496625:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205067.496628:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496631:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205067.496635:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205067.496637:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205067.496640:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205067.496642:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205067.496644:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205067.496646:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205067.496650:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496651:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205067.496653:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496654:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205067.496657:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205067.496660:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496662:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205067.496665:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205067.496668:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205067.496671:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.496673:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205067.496676:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205067.496685:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205067.496688:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205067.496693:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205067.496694:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.496697:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205067.496699:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205067.496703:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205067.496711:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788205067.496717:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205067.496726:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205067.496736:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205067.496740:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205067.496745:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205067.496769:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205067.496772:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205067.496775:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205067.496777:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205067.496778:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205067.496781:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205067.496783:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6522 00000400:00000200:2.0:1788205067.496787:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205067.496790:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205067.496793:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205067.496796:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496798:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205067.496800:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496803:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205067.496805:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205067.496806:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1788205067.496807:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:0.0:1788205067.496810:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788205067.496811:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205067.496814:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205067.496817:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205067.496821:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788205067.496821:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205067.496827:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496830:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205067.496832:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205067.496833:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.496839:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205067.496844:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205067.496850:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205067.496852:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205067.496855:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.496856:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205067.496860:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205067.496863:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205067.496867:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205067.496867:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205067.496872:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205067.496873:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.496878:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1788205067.496879:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205925504:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205067.496884:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788205067.496885:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788205067.496888:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205067.496891:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205067.496891:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.496897:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205067.496897:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205067.496903:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 00000100:00000010:2.0:1788205067.496904:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000001:0.0:1788205067.496907:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205067.496909:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:0.0:1788205067.496914:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205067.496915:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205067.496918:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205067.496919:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000100:00000001:2.0:1788205067.496921:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205067.496924:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205067.496927:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205067.496929:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 02000000:00000010:2.0:1788205067.496932:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000200:0.0:1788205067.496937:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925504, portal 25 00000100:00000001:2.0:1788205067.496938:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.496943:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205067.496947:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00100000:2.0:1788205067.496949:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205067.496960:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205067.496963:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205067.496965:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205067.496971:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.496976:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205067.496983:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205067.496993:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000001:2.0:1788205067.496995:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205067.496998:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205925504, offset 0 00000100:00080000:2.0:1788205067.497000:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205067.497005:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205067.497009:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1788205067.497010:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205067.497012:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205067.497015:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205067.497019:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6522 00000400:00000200:0.0:1788205067.497021:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205067.497026:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.497031:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205067.497032:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205067.497038:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1788205067.497042:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788205067.497044:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497048:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:2.0:1788205067.497050:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205067.497053:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1788205067.497057:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497058:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497062:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205067.497064:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59086:59086:256:4294967295] 192.168.201.151@tcp LPNI seq info [59066:59066:8:4294967295] 00000020:00000001:2.0:1788205067.497066:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205067.497070:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:0.0:1788205067.497071:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:2.0:1788205067.497074:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205067.497077:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:0.0:1788205067.497083:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:2.0:1788205067.497083:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205067.497088:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:0.0:1788205067.497093:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788205067.497095:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000010:0.0:1788205067.497098:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000100:00000001:2.0:1788205067.497100:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205067.497104:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:0.0:1788205067.497108:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205067.497110:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:0.0:1788205067.497112:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000020:00000040:2.0:1788205067.497114:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000200:0.1:1788205067.497120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00080000:2.0:1788205067.497123:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:0.1:1788205067.497124:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788205067.497130:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788205067.497131:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205067.497136:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:0.0:1788205067.497137:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 02000000:00000001:2.0:1788205067.497142:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205067.497144:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1788205067.497148:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205067.497150:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205067.497153:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1788205067.497155:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205067.497158:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000008:00100000:2.0:1788205067.497158:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:0.0:1788205067.497161:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1788205067.497163:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.497167:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205067.497171:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205067.497174:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000200:0.0:1788205067.497175:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205067.497177:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205067.497180:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.497184:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205067.497186:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205067.497188:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205067.497194:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:0.0:1788205067.497197:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205067.497199:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205067.497204:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205067.497209:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205067.497224:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.497225:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205067.497226:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205067.497229:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205067.497233:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497234:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205067.497242:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205067.497246:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301198849 00000100:00000001:2.0:1788205067.497248:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205067.497253:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00080000:2.0:1788205067.497253:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205067.497260:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205067.497261:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205067.497263:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788205067.497263:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205067.497269:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205067.497272:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301198849 00000100:00080000:2.0:1788205067.497273:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6522 00000100:00000010:0.0:1788205067.497279:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9900. 00000400:00000200:2.0:1788205067.497281:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205067.497286:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000400:00000200:2.0:1788205067.497287:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205067.497292:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205067.497294:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788205067.497295:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497299:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205067.497301:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205067.497302:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497306:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205067.497308:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788205067.497308:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205067.497311:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205067.497313:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205067.497315:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205067.497317:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00100000:0.0:1788205067.497319:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205067/real 1788205067] req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205067.497319:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205067.497325:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497327:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205067.497328:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497330:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205067.497332:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000001:0.0:1788205067.497344:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205067.497346:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205067.497348:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205067.497351:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205067.497353:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205067.497355:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205067.497417:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205067.497420:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205067.497425:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205067.497428:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205067.497431:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205067.497435:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205067.497437:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205067.497437:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205067.497440:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205067.497444:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.497448:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205067.497454:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205067.497455:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1788205067.497457:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:0.0:1788205067.497459:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205067.497460:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205067.497462:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497465:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205067.497465:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205067.497466:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788205067.497470:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205067.497470:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205067.497476:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00100000:0.0:1788205067.497480:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205067.497481:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205067.497488:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205067.497504:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205067.497504:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205067.497508:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205067.497518:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205067.497519:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205067.497531:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205067.497536:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6532) 00000100:00000001:0.0:1788205067.497546:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205067.497550:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497552:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.497553:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497556:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205067.497557:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205067.497559:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205067.497562:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205067.497564:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205067.497567:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205067.497573:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205925504:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205067.497577:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497579:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.497582:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205925504/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497588:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205067.497591:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205067.497593:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205067.497597:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205067.497599:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205067.497601:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205067.497603:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205067.497605:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:0.0:1788205067.497609:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205067.497611:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205067.497613:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205067.497615:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205067.497616:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497617:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497623:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205067.497624:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497625:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205067.497626:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205067.497627:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205067.497630:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497634:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205067.497635:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.497636:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205067.497638:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205067.497639:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205067.497640:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925632:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205067.497642:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205067.497644:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205067.497645:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.497646:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205067.497648:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205067.497649:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205067.497652:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205067.497654:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205067.497656:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205067.497658:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925632, portal 10 00000100:00000001:0.0:1788205067.497659:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205067.497660:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205067.497664:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497668:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205067.497670:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205067.497671:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205067.497672:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205925632, offset 0 00000400:00000200:0.0:1788205067.497675:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497678:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.497682:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205067.497685:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205067.497687:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205067.497690:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205067.497692:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497695:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59087:59087:256:4294967295] 192.168.201.151@tcp LPNI seq info [59067:59067:8:4294967295] 00000400:00000200:0.0:1788205067.497699:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.497704:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205067.497709:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.497712:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205067.497716:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205067.497719:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205067.497723:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205067.497726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205067.497730:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.497731:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205067.497734:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205067.497736:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205067.497738:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205067.497739:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205067.497741:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205067.497744:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497748:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.497750:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497754:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497755:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205067.497757:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205067.497760:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205067.497762:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205067.497763:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301194907 00000100:00000001:0.0:1788205067.497765:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.497767:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301194907 00000100:00000001:0.0:1788205067.497768:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.497769:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205067.497771:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497772:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205067.497774:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205067/real 1788205067] req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205067.497778:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205067.497779:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205067.497780:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205067.497781:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205067.497782:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205067.497783:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205067.497785:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205067.497790:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497793:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205067.497794:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497795:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.497795:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205067.497797:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497801:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205067.497803:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497807:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205067.497808:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497810:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.497811:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.497812:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205067.497815:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205067.497817:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205067.497820:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205067.497822:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205067.497826:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205067.497833:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925632:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205067.497837:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205067.497840:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497846:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205067.497848:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.497850:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205067.497852:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205067.497854:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205067.497857:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205067.497860:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205067.497861:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205067.497862:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.497863:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205067.497866:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205067.497867:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205067.497869:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205067.497871:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205067.497873:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205067.497875:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925760, portal 4 00000100:00000001:0.0:1788205067.497876:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205067.497877:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205067.497881:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497885:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205067.497887:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205067.497888:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205067.497889:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205925760, offset 0 00000400:00000200:0.0:1788205067.497891:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497894:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.497896:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205067.497897:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205067.497899:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205067.497901:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205067.497902:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205067.497904:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59088:59088:256:4294967295] 192.168.201.151@tcp LPNI seq info [59068:59068:8:4294967295] 00000400:00000200:0.0:1788205067.497906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.497910:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205067.497913:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.497914:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205067.497917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205067.497919:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205067.497923:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205067.497925:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205067.497929:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.497931:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205067.497933:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205067.497936:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205067.497939:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205067.497941:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205067.497943:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205067.497946:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497953:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.497956:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.497962:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497964:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205067.497967:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205067.497969:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205067.497971:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205067.497973:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301194793 00000100:00000001:0.0:1788205067.497974:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.497976:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301194793 00000100:00000001:0.0:1788205067.497977:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.497978:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205067.497980:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.497981:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205067.497983:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205067/real 1788205067] req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205067.497988:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205067.497989:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205067.497990:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205067.497993:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205067.497994:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205067.497996:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205067.497998:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205067.498006:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498012:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205067.498013:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498015:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.498016:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205067.498020:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498026:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205067.498030:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498034:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205067.498036:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498037:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.498037:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498038:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205067.498040:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205067.498041:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205067.498043:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205067.498044:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205067.498046:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205067.498051:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205067.498053:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205067.498055:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498058:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205067.498059:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.498060:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205067.498061:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205067.498063:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205067.498064:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925888:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205067.498066:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205067.498068:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205067.498069:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205067.498070:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205067.498072:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788205067.498073:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205067.498075:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205067.498076:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205067.498078:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205067.498080:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205925888, portal 4 00000100:00000001:0.0:1788205067.498081:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205067.498082:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205067.498085:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498089:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205067.498090:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205067.498092:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205067.498093:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205925888, offset 0 00000400:00000200:0.0:1788205067.498095:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205067.498097:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.498099:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205067.498101:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205067.498103:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205067.498104:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205067.498106:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205067.498107:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59089:59089:256:4294967295] 192.168.201.151@tcp LPNI seq info [59069:59069:8:4294967295] 00000400:00000200:0.0:1788205067.498110:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205067.498113:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205067.498116:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.498118:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205067.498120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205067.498122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205067.498124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205067.498126:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205067.498128:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205067.498130:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205067.498132:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205067.498134:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205067.498135:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205067.498136:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205067.498138:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205067.498140:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498144:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.498146:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205092 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498149:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.498150:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205067.498152:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205067.498153:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205067.498155:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205067.498156:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301194799 00000100:00000001:0.0:1788205067.498158:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.498159:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301194799 00000100:00000001:0.0:1788205067.498160:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301198849 00000100:00000001:0.0:1788205067.498162:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205067.498163:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.498164:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205067.498166:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205067/real 1788205067] req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205067.498172:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205067.498173:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205067.498175:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205067.498177:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205067.498179:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205067.498181:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205067.498183:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205067.498191:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498197:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205067.498198:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498200:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.498201:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205067.498205:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498211:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205067.498213:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498218:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205067.498219:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498220:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205067.498221:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498222:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205067.498223:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205067.498225:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205067.498226:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205067.498227:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205067.498229:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205067.498233:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205925888:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205067.498236:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498237:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.498239:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205925632/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498243:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205067.498244:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205067.498245:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205067.498247:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205067.498248:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205067.498248:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205067.498250:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205067.498251:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:0.0:1788205067.498252:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205067.498253:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205067.498254:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205067.498256:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205067.498256:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498257:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.498259:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205925760/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498263:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205067.498263:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205067.498264:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205067.498265:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205067.498266:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205067.498267:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205067.498268:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205067.498269:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:0.0:1788205067.498270:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205067.498271:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205067.498271:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205067.498272:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205067.498273:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498274:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205067.498275:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205925888/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205092 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205067.498279:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205067.498280:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205067.498280:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788205067.498282:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205067.498283:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205067.498284:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205067.498286:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205067.498287:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205067.498289:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205067.498290:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205067.498291:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205067.498293:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205067.498294:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498295:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205067.498299:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205067.498300:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205067.498302:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205067.498303:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205067.688577:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205067.688584:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205067.688590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205067.688591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205067.688594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205067.688595:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205067.688598:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205067.688599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205067.688659:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205067.688662:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205067.688665:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205068.008594:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205068.008602:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205068.008604:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205068.008609:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205068.008612:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205068.008617:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205068.008619:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205068.008622:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205068.008624:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205068.456534:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205068.456542:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205068.456543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205068.456548:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205068.456553:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205068.456557:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205068.456559:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205068.456562:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205068.456564:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205068.520485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205068.520492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205068.520494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205068.520499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205068.520503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205068.520506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205068.520508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205068.520511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205068.520514:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205068.522442:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205068.522447:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205068.522449:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205068.522453:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205068.522456:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205068.522459:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205068.522461:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205068.522463:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205068.522466:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205068.712488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205068.712496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205068.712501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205068.712502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205068.712505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205068.712506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205068.712509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205068.712510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205068.712566:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205068.712570:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205068.712574:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205069.032470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205069.032476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205069.032478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205069.032481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205069.032484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205069.032487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205069.032489:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205069.032491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205069.032492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.480453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205069.480458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.480460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.480464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205069.480467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205069.480470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.480472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.480473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.480475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.544463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205069.544470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.544472:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.544478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205069.544481:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205069.544484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.544487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.544491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.544494:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205069.546439:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205069.546443:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205069.546445:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205069.546449:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205069.546453:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205069.546456:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205069.546458:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205069.546460:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205069.546463:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205069.736510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205069.736515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205069.736520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205069.736521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205069.736524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205069.736525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205069.736528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205069.736529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205069.736582:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205069.736591:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205069.736595:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205070.056442:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205070.056449:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205070.056452:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205070.056457:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205070.056462:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205070.056465:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205070.056468:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205070.056471:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205070.056475:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205070.504457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205070.504462:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205070.504463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205070.504467:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205070.504470:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205070.504473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205070.504474:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205070.504476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205070.504478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205070.568470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205070.568470:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205070.568477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205070.568478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205070.568479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205070.568481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205070.568486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205070.568487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205070.568489:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205070.568493:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205070.568493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205070.568497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205070.568498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205070.568501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205070.568505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205070.568510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205070.568513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205070.568516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205070.760478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205070.760484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205070.760489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205070.760491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205070.760493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205070.760495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205070.760498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205070.760499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205070.760565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205070.760569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205070.760573:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205071.080496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205071.080504:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205071.080506:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205071.080511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205071.080514:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205071.080519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205071.080521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205071.080524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205071.080526:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205071.528479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205071.528485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205071.528487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205071.528491:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205071.528494:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205071.528497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205071.528499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205071.528501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205071.528503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205071.592449:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205071.592456:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205071.592458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205071.592463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205071.592467:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205071.592471:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205071.592474:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205071.592478:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205071.592481:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205071.592530:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205071.592534:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205071.592535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205071.592538:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205071.592540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205071.592543:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205071.592545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205071.592546:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205071.592548:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205071.784554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205071.784562:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205071.784564:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205071.784568:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784571:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205071.784573:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784574:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784576:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205071.784579:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205071.784581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205071.784582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205071.784712:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205071.784716:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205071.784719:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205072.104473:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301198849, at 4301199360 00000100:00000040:1.0:1788205072.104473:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301198849, at 4301199360 00000100:00000001:2.0:1788205072.104474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205072.104479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205072.104484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205072.104487:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9900. 00000100:00100000:2.0:1788205072.104490:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205072.104493:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:2.0:1788205072.104495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205072.104500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205072.104502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205072.104505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205072.104509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.552464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205072.552470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205072.552472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205072.552477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205072.552481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205072.552485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205072.552487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205072.552489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205072.552493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.616446:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205072.616447:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205072.616453:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205072.616456:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.616458:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205072.616461:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205072.616463:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205072.616463:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205072.616465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205072.616470:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205072.616471:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205072.616472:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205072.616475:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205072.616476:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205072.616477:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.616480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205072.616481:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205072.616485:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205072.616486:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205072.616490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205072.616491:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6527 00000100:00000001:3.0:1788205072.616491:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.616493:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205072.616498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205072.616503:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205072.616508:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205072.616510:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205072.616514:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616516:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.616518:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616520:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205072.616522:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205072.616523:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205072.616525:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205072.616527:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205072.616529:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205072.616530:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205072.616532:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616533:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.616534:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616535:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205072.616537:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205072.616539:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616540:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205072.616542:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205072.616545:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205072.616546:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.616547:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205072.616549:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205072.616556:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205072.616558:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205072.616561:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205072.616563:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.616565:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205072.616567:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205072.616570:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205072.616576:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788205072.616580:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205072.616587:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205072.616594:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205072.616597:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205072.616600:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205072.616619:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205072.616621:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205072.616624:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205072.616626:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205072.616627:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205072.616630:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205072.616632:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6527 00000400:00000200:0.0:1788205072.616635:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205072.616638:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205072.616640:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205072.616644:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616646:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.616649:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616651:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205072.616653:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205072.616655:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788205072.616656:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205072.616657:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205072.616662:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788205072.616662:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205072.616667:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205072.616668:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205072.616672:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205072.616676:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205072.616678:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616683:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.616688:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616701:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205072.616702:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205072.616705:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205072.616706:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205072.616711:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788205072.616714:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788205072.616715:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.616718:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205072.616719:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205072.616724:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205072.616726:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788205072.616729:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205926016:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788205072.616731:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.616735:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788205072.616736:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205072.616739:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788205072.616742:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205072.616742:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.616746:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788205072.616748:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205072.616751:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205072.616754:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.616755:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.616759:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:0.0:1788205072.616762:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000010:3.0:1788205072.616764:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000020:00000040:0.0:1788205072.616768:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:3.0:1788205072.616770:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 02000000:00000001:0.0:1788205072.616772:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205072.616775:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.616777:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1788205072.616777:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926016, portal 25 02000000:00000001:0.0:1788205072.616780:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205072.616781:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788205072.616784:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.616784:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205072.616791:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788205072.616792:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788205072.616798:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.616805:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788205072.616808:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205072.616812:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616821:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1788205072.616822:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788205072.616824:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205072.616827:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205926016, offset 0 00000100:00000040:0.0:1788205072.616831:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205072.616839:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205072.616850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205072.616852:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205072.616856:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205072.616859:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205072.616860:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205072.616862:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205072.616863:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788205072.616866:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205072.616869:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1788205072.616872:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6527 00000400:00000200:3.0:1788205072.616876:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205072.616878:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.616881:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205072.616885:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205072.616886:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:0.0:1788205072.616891:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205072.616891:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59090:59090:256:4294967295] 192.168.201.151@tcp LPNI seq info [59070:59070:8:4294967295] 00000100:00000040:0.0:1788205072.616897:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616901:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205072.616903:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788205072.616905:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616908:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205072.616911:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205072.616913:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205072.616916:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205072.616919:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788205072.616920:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000001:0.0:1788205072.616923:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205072.616927:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.0:1788205072.616931:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788205072.616933:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616937:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.616939:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000010:3.0:1788205072.616939:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:0.0:1788205072.616944:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205072.616945:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205072.616950:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000800:00000200:3.0:1788205072.616951:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205072.616953:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205072.616955:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 02000000:00000001:0.0:1788205072.616957:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788205072.616961:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:0.0:1788205072.616962:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1788205072.616964:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 02000000:00000001:0.0:1788205072.616968:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205072.616969:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205072.616974:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:3.0:1788205072.616974:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205072.616978:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:0.0:1788205072.616979:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205072.616985:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.616985:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788205072.616989:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.616990:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205072.616993:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000010:0.0:1788205072.616995:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205072.616996:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1788205072.617000:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000200:3.0:1788205072.617002:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205072.617007:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205072.617010:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617012:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205072.617014:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1788205072.617016:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205072.617018:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205072.617022:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205072.617030:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:3.0:1788205072.617030:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617034:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205072.617036:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617038:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.617044:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:0.0:1788205072.617046:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205072.617046:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301203969 00000100:00000010:3.0:1788205072.617052:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4480. 00000100:00000001:3.0:1788205072.617059:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205072.617060:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205072.617064:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617065:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301203969 00000100:00000010:3.0:1788205072.617069:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c00. 00000100:00000040:0.0:1788205072.617070:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617072:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205072.617076:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205072.617081:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617085:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205072.617090:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205072.617091:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205072/real 1788205072] req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205072.617095:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205072.617104:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000001:3.0:1788205072.617105:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788205072.617106:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1788205072.617108:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1788205072.617110:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205072.617110:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205072.617114:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000001:0.0:1788205072.617116:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788205072.617116:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00080000:0.0:1788205072.617120:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6527 00000400:00000200:0.0:1788205072.617126:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205072.617128:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205072.617132:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205072.617134:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205072.617140:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788205072.617145:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205072.617147:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.617152:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.617156:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.617161:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205072.617162:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1788205072.617164:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788205072.617165:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788205072.617166:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205072.617170:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205072.617171:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1788205072.617172:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205072.617174:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205072.617176:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205072.617180:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00100000:3.0:1788205072.617183:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205072.617185:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.617190:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205072.617194:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205072.617198:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205072.617201:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788205072.617202:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205072.617207:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:3.0:1788205072.617211:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205072.617212:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205072.617216:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205072.617220:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205072.617224:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.617227:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205072.617227:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:0.0:1788205072.617230:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205072.617233:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788205072.617234:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205072.617236:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205072.617239:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205072.617241:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205072.617246:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205072.617247:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788205072.617248:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617250:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205072.617251:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205072.617254:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205072.617255:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1788205072.617256:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617259:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:0.0:1788205072.617261:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00080000:3.0:1788205072.617264:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788205072.617267:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205072.617280:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00100000:0.0:1788205072.617285:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205072.617286:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788205072.617296:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205072.617298:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205072.617302:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205072.617306:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205072.617327:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205072.617330:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205926016:192.168.201.151@tcp:250:kworker.0 00000100:00000040:0.0:1788205072.617332:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6537) 00000100:00000001:3.0:1788205072.617339:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617341:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.617345:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205926016/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617352:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205072.617355:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205072.617357:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205072.617360:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205072.617362:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205072.617382:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205072.617385:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205072.617387:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788205072.617390:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205072.617392:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205072.617394:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205072.617397:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205072.617399:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617401:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617407:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205072.617408:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617410:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205072.617411:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205072.617413:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205072.617416:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617423:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205072.617425:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617427:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205072.617428:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205072.617430:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205072.617433:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926144:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205072.617436:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205072.617438:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205072.617439:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617441:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205072.617444:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205072.617446:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.617449:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205072.617452:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205072.617455:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205072.617458:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926144, portal 10 00000100:00000001:3.0:1788205072.617460:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205072.617461:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205072.617468:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617474:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205072.617476:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205072.617479:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205072.617482:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205926144, offset 0 00000400:00000200:3.0:1788205072.617486:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617489:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617493:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205072.617496:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205072.617498:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205072.617500:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205072.617502:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617505:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59091:59091:256:4294967295] 192.168.201.151@tcp LPNI seq info [59071:59071:8:4294967295] 00000400:00000200:3.0:1788205072.617509:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617515:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205072.617519:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.617522:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205072.617526:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205072.617529:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205072.617534:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205072.617536:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205072.617540:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.617541:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205072.617544:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205072.617547:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205072.617551:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205072.617552:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205072.617555:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205072.617558:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617566:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.617568:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617573:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617574:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205072.617577:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205072.617579:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205072.617581:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205072.617583:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301200027 00000100:00000001:3.0:1788205072.617584:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.617586:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301200027 00000100:00000001:3.0:1788205072.617587:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.617588:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205072.617590:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617591:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205072.617593:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205072/real 1788205072] req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205072.617598:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205072.617599:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205072.617600:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205072.617601:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205072.617602:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205072.617604:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205072.617605:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205072.617610:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617613:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205072.617614:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617615:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.617616:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205072.617618:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617621:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205072.617623:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617628:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205072.617629:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617630:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.617631:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617632:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205072.617634:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205072.617635:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205072.617637:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205072.617638:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205072.617640:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205072.617644:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926144:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205072.617646:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205072.617647:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617652:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205072.617652:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617654:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205072.617655:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205072.617656:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205072.617657:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205072.617659:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205072.617660:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205072.617661:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617662:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205072.617664:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205072.617665:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.617667:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205072.617668:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205072.617670:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205072.617672:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926272, portal 4 00000100:00000001:3.0:1788205072.617673:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205072.617674:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205072.617678:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617681:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205072.617683:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205072.617684:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205072.617685:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205926272, offset 0 00000400:00000200:3.0:1788205072.617687:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617689:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617691:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205072.617693:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205072.617696:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205072.617698:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205072.617700:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617703:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59092:59092:256:4294967295] 192.168.201.151@tcp LPNI seq info [59072:59072:8:4294967295] 00000400:00000200:3.0:1788205072.617706:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617712:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205072.617716:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.617720:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205072.617723:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205072.617726:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205072.617730:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205072.617732:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205072.617736:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.617738:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205072.617740:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205072.617744:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205072.617746:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205072.617748:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205072.617750:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205072.617754:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617760:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.617764:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617770:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617771:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205072.617774:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205072.617777:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205072.617780:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205072.617782:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301199913 00000100:00000001:3.0:1788205072.617784:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.617786:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301199913 00000100:00000001:3.0:1788205072.617788:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.617789:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205072.617791:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.617793:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205072.617796:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205072/real 1788205072] req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205072.617803:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205072.617805:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205072.617806:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205072.617808:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205072.617809:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205072.617811:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205072.617814:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205072.617821:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617832:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205072.617833:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617835:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.617836:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205072.617840:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617846:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205072.617849:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617855:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205072.617857:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617859:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.617860:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.617862:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205072.617863:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205072.617866:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205072.617869:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205072.617871:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205072.617875:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205072.617882:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205072.617885:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205072.617888:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617894:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205072.617896:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617898:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205072.617900:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205072.617902:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205072.617904:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926400:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205072.617907:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205072.617909:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205072.617910:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205072.617912:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205072.617915:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205072.617917:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205072.617920:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205072.617922:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205072.617925:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205072.617927:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926400, portal 4 00000100:00000001:3.0:1788205072.617929:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205072.617930:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788205072.617935:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.617940:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205072.617942:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205072.617944:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205072.617946:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205926400, offset 0 00000400:00000200:3.0:1788205072.617949:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617952:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617955:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205072.617958:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205072.617960:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205072.617962:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205072.617965:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205072.617968:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59093:59093:256:4294967295] 192.168.201.151@tcp LPNI seq info [59073:59073:8:4294967295] 00000400:00000200:3.0:1788205072.617972:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205072.617977:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205072.617982:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.617985:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205072.617988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205072.617991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205072.617995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205072.617997:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205072.618001:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205072.618002:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205072.618005:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205072.618009:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205072.618011:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205072.618013:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205072.618015:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205072.618019:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618025:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.618028:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618034:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.618035:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205072.618038:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205072.618040:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205072.618043:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205072.618045:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301199919 00000100:00000001:3.0:1788205072.618047:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.618049:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301199919 00000100:00000001:3.0:1788205072.618051:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301203969 00000100:00000001:3.0:1788205072.618053:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205072.618055:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.618057:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205072.618060:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205072/real 1788205072] req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205072.618068:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205072.618069:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205072.618070:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205072.618072:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205072.618074:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205072.618075:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205072.618078:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205072.618085:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618091:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205072.618092:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618094:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.618095:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205072.618098:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618104:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205072.618108:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618113:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205072.618116:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618117:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205072.618118:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618120:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205072.618122:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205072.618124:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205072.618127:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205072.618129:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205072.618133:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205072.618140:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926400:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205072.618143:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618145:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.618148:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205926144/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618154:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205072.618155:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205072.618157:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205072.618159:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205072.618161:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205072.618162:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205072.618164:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205072.618166:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788205072.618169:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205072.618170:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205072.618171:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205072.618173:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205072.618175:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618176:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.618179:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205926272/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618185:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205072.618186:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205072.618187:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205072.618189:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205072.618190:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205072.618191:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205072.618193:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205072.618194:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788205072.618196:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205072.618197:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205072.618198:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205072.618200:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205072.618202:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618203:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205072.618206:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205926400/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205072.618212:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205072.618213:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205072.618214:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205072.618216:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205072.618217:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205072.618218:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205072.618219:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205072.618221:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788205072.618222:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205072.618223:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205072.618224:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205072.618226:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205072.618227:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618229:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205072.618232:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205072.618234:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205072.618235:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205072.618237:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205072.808449:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205072.808454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205072.808459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205072.808461:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205072.808463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205072.808464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205072.808466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205072.808467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205072.808514:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205072.808517:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205072.808520:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205073.128461:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205073.128466:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205073.128468:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205073.128471:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205073.128474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205073.128477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205073.128479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205073.128481:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205073.128482:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.576460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205073.576465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205073.576466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205073.576470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205073.576472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205073.576475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205073.576477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205073.576478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205073.576480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.640467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205073.640474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205073.640477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205073.640482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205073.640486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205073.640488:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205073.640489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205073.640496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205073.640497:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205073.640503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205073.640503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.640515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205073.640515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205073.640549:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205073.640552:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205073.640555:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205073.640557:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205073.640559:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205073.832588:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205073.832591:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205073.832594:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832598:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205073.832600:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832602:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832605:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832609:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205073.832610:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832613:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205073.832615:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205073.832616:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205073.832617:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832619:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205073.832620:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832622:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832632:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832638:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205073.832640:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205073.832641:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205073.832643:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832644:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205073.832645:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832647:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205073.832649:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205073.832715:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205073.832719:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205073.832722:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205074.152488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205074.152496:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205074.152497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205074.152503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205074.152506:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205074.152509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205074.152512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205074.152514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205074.152516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.600490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205074.600498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.600499:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.600504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205074.600507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205074.600511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.600513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.600515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.600517:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.664500:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205074.664511:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.664515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.664522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205074.664527:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205074.664532:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.664535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.664538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.664541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205074.664542:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205074.664548:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205074.664549:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205074.664553:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205074.664594:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205074.664642:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205074.664645:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205074.664646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205074.664648:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205074.856502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205074.856512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205074.856519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205074.856521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205074.856525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205074.856526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205074.856531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205074.856533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205074.856609:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205074.856614:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205074.856619:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205075.176495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205075.176506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205075.176508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205075.176514:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205075.176517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205075.176521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205075.176524:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205075.176526:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205075.176528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205075.624526:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205075.624545:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205075.624548:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205075.624555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205075.624560:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205075.624565:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.624568:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205075.624570:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.624573:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205075.688486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205075.688486:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205075.688496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205075.688496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205075.688499:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205075.688500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205075.688506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205075.688507:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205075.688513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205075.688513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205075.688518:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205075.688518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.688522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205075.688527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.688532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205075.688532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205075.688535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205075.688539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205075.880545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205075.880553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205075.880560:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205075.880561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205075.880565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205075.880566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205075.880570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205075.880571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205075.880679:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205075.880684:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205075.880688:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205076.200569:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205076.200585:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205076.200587:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205076.200594:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205076.200598:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205076.200602:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205076.200605:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205076.200609:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205076.200612:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205076.648501:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205076.648513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205076.648515:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205076.648522:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205076.648527:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205076.648531:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.648534:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205076.648535:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.648538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205076.712489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205076.712489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205076.712501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205076.712501:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205076.712506:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205076.712506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205076.712515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205076.712520:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788205076.712524:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205076.712526:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.712529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205076.712531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205076.712533:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205076.712536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205076.712541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205076.712544:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205076.712548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.712551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205076.904504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205076.904513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205076.904521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205076.904523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205076.904527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205076.904528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205076.904532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205076.904534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205076.904636:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205076.904639:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205076.904641:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205077.224500:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301203969, at 4301204480 00000100:00000040:1.0:1788205077.224500:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301203969, at 4301204480 00000100:00000001:2.0:1788205077.224501:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205077.224508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205077.224512:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205077.224517:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205077.224518:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000001:2.0:1788205077.224521:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205077.224527:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205077.224530:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205077.224533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205077.224537:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205077.224547:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000001:1.0:1788205077.672508:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205077.672518:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205077.672520:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205077.672535:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205077.672539:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205077.672543:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205077.672546:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205077.672548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205077.672551:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205077.736470:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205077.736470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205077.736478:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788205077.736483:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205077.736483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736488:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788205077.736491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:0.0:1788205077.736495:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205077.736496:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205077.736497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205077.736503:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205077.736504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.736505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736507:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1788205077.736510:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205077.736514:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205077.736518:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6532 00000100:00000001:1.0:1788205077.736519:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205077.736524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.736525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205077.736531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.736531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1788205077.736534:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205077.736535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.736540:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205077.736547:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205077.736552:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205077.736556:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736560:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736564:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736567:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205077.736571:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205077.736574:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205077.736576:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205077.736580:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205077.736582:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205077.736585:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205077.736588:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736590:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736592:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736594:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205077.736597:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205077.736600:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736603:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205077.736607:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205077.736611:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205077.736614:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736617:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205077.736620:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205077.736633:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205077.736636:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205077.736639:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205077.736641:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736645:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205077.736648:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205077.736653:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205077.736665:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788205077.736673:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205077.736684:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205077.736694:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205077.736698:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205077.736702:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205077.736721:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205077.736723:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205077.736726:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205077.736728:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205077.736729:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205077.736731:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205077.736733:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6532 00000400:00000200:0.0:1788205077.736736:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205077.736738:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205077.736740:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205077.736743:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736744:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736746:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736748:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205077.736749:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205077.736750:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788205077.736750:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205077.736753:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205077.736755:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788205077.736756:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205077.736759:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788205077.736759:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205077.736763:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205077.736767:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788205077.736768:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205077.736770:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736774:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736778:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205077.736782:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788205077.736784:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205077.736786:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205077.736787:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736789:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205077.736791:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205077.736792:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205077.736795:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205077.736797:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205077.736797:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205077.736801:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205926528:192.168.201.151@tcp:250:kworker.0 02000000:00000001:0.0:1788205077.736802:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736806:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205077.736810:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205077.736813:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788205077.736814:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736817:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.736819:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1788205077.736820:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 02000000:00000001:3.0:1788205077.736822:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205077.736823:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205077.736826:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788205077.736827:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205077.736829:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736830:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205077.736831:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:3.0:1788205077.736832:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22000. 02000000:00000001:0.0:1788205077.736833:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.736837:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205077.736840:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788205077.736844:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.736845:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205077.736850:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00100000:0.0:1788205077.736851:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788205077.736857:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788205077.736858:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205077.736862:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205077.736865:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205077.736869:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926528, portal 25 00000100:00000001:0.0:1788205077.736870:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.736872:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788205077.736873:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205077.736875:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00080000:0.0:1788205077.736880:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205077.736883:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:3.0:1788205077.736884:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205077.736885:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205077.736891:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205077.736896:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6532 00000100:00000001:3.0:1788205077.736898:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1788205077.736901:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:3.0:1788205077.736905:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205077.736908:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:3.0:1788205077.736908:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00080000:0.0:1788205077.736911:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788205077.736913:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205926528, offset 0 00000100:00000040:0.0:1788205077.736915:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736918:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736920:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736921:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205077.736924:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205077.736926:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205077.736926:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000001:0.0:1788205077.736928:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205077.736930:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205077.736932:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205077.736933:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205077.736935:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736936:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.736937:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736941:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205077.736941:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:0.0:1788205077.736943:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205077.736946:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205077.736948:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205077.736949:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205077.736951:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205077.736953:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.736953:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788205077.736957:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205077.736960:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205077.736962:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.736963:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205077.736964:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205077.736968:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205077.736971:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788205077.736971:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:0.0:1788205077.736975:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788205077.736976:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1788205077.736977:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.736980:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788205077.736982:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:0.0:1788205077.736983:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205077.736986:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205077.736988:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000400:00000200:3.0:1788205077.736989:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59094:59094:256:4294967295] 192.168.201.151@tcp LPNI seq info [59074:59074:8:4294967295] 00000100:00000001:0.0:1788205077.736991:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.736996:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:0.0:1788205077.736997:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205077.737003:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205077.737004:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205077.737010:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205077.737016:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788205077.737018:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205077.737020:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205077.737024:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205077.737025:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205077.737026:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205077.737030:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.0:1788205077.737030:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205077.737035:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6532 00000400:00000200:0.0:1788205077.737039:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000010:3.0:1788205077.737040:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000400:00000200:0.0:1788205077.737043:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205077.737047:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205077.737052:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:3.0:1788205077.737052:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205077.737058:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788205077.737060:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00000040:0.0:1788205077.737062:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:3.1:1788205077.737072:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205077.737077:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.1:1788205077.737077:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1788205077.737085:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:3.0:1788205077.737085:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788205077.737090:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000010:3.0:1788205077.737090:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000020:00000001:0.0:1788205077.737095:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205077.737098:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:3.0:1788205077.737100:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1788205077.737101:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205077.737106:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:3.0:1788205077.737107:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1788205077.737114:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737114:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205077.737118:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:0.0:1788205077.737119:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205077.737124:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205077.737124:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788205077.737129:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:3.0:1788205077.737131:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205077.737133:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205077.737142:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205077.737144:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205077.737146:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205077.737149:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205077.737150:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205077.737155:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205077.737157:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737160:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1788205077.737161:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.737164:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205077.737167:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205077.737167:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.737172:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205077.737175:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301209089 00000100:00000010:3.0:1788205077.737179:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f42a0. 00000008:00000001:0.0:1788205077.737196:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.737199:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737203:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000010:0.0:1788205077.737204:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205077.737209:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205077.737211:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205077.737212:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205077.737214:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205077.737214:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301209089 02000000:00000001:0.0:1788205077.737217:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205077.737220:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205077.737221:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4480. 02000000:00000010:0.0:1788205077.737225:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000100:00000001:3.0:1788205077.737227:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205077.737231:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737234:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788205077.737239:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205077.737239:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737244:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205077.737250:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788205077.737253:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205077/real 1788205077] req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205077.737254:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205077.737263:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205077.737292:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205077.737294:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205077.737297:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1788205077.737298:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6542) 00000400:00000010:3.0:1788205077.737300:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205077.737304:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205077.737306:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205077.737309:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205077.737314:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205077.737323:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737330:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205077.737332:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737335:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737337:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205077.737341:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737349:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205077.737353:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737359:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205077.737362:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737383:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737385:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737387:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205077.737389:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205077.737392:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205077.737396:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205077.737398:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205077.737403:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205077.737412:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205926528:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205077.737417:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737419:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.737422:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205926528/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737428:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205077.737431:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205077.737432:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205077.737435:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205077.737436:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205077.737437:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205077.737439:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205077.737441:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788205077.737444:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205077.737446:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205077.737448:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205077.737450:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205077.737451:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737452:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737459:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205077.737460:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737461:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205077.737462:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205077.737463:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205077.737466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737470:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205077.737471:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737473:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205077.737474:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205077.737475:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205077.737477:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926656:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205077.737480:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205077.737482:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205077.737483:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737485:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205077.737488:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205077.737490:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.737494:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205077.737496:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205077.737500:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205077.737503:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926656, portal 10 00000100:00000001:3.0:1788205077.737505:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205077.737506:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205077.737513:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737520:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205077.737522:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205077.737525:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205077.737527:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205926656, offset 0 00000400:00000200:3.0:1788205077.737531:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737535:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.737538:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205077.737541:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205077.737544:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205077.737546:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205077.737549:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737551:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59095:59095:256:4294967295] 192.168.201.151@tcp LPNI seq info [59075:59075:8:4294967295] 00000400:00000200:3.0:1788205077.737555:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.737562:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205077.737566:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.737570:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205077.737573:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205077.737577:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205077.737580:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205077.737582:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205077.737584:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.737585:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205077.737587:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205077.737590:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205077.737592:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205077.737593:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205077.737595:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205077.737597:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737602:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.737604:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737608:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737609:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205077.737611:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205077.737614:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205077.737616:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205077.737617:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301205147 00000100:00000001:3.0:1788205077.737619:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.737621:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301205147 00000100:00000001:3.0:1788205077.737622:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.737623:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205077.737625:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737626:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205077.737628:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205077/real 1788205077] req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205077.737632:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205077.737640:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205077.737642:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205077.737643:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205077.737644:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205077.737646:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205077.737648:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205077.737653:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737656:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205077.737657:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737658:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737659:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205077.737661:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737664:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205077.737666:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737671:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205077.737674:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737675:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737676:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737678:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205077.737680:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205077.737682:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205077.737686:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205077.737689:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205077.737692:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205077.737700:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926656:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205077.737705:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205077.737708:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737715:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205077.737716:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737718:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205077.737720:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205077.737722:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205077.737725:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205077.737728:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205077.737730:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205077.737731:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737732:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205077.737735:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205077.737737:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.737739:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205077.737741:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205077.737743:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205077.737745:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926784, portal 4 00000100:00000001:3.0:1788205077.737746:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205077.737747:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205077.737751:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737755:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205077.737757:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205077.737758:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205077.737760:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205926784, offset 0 00000400:00000200:3.0:1788205077.737762:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737764:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.737766:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205077.737768:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205077.737770:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205077.737771:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205077.737773:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737775:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59096:59096:256:4294967295] 192.168.201.151@tcp LPNI seq info [59076:59076:8:4294967295] 00000400:00000200:3.0:1788205077.737777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.737781:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205077.737783:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.737785:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205077.737787:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205077.737789:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205077.737792:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205077.737808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205077.737811:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.737813:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205077.737815:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205077.737817:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205077.737819:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205077.737820:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205077.737822:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205077.737825:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737832:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.737835:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737841:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737842:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205077.737845:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205077.737849:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205077.737852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205077.737854:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301205033 00000100:00000001:3.0:1788205077.737856:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.737858:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301205033 00000100:00000001:3.0:1788205077.737861:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.737863:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205077.737866:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.737868:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205077.737871:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205077/real 1788205077] req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205077.737877:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205077.737878:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205077.737879:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205077.737881:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205077.737882:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205077.737883:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205077.737885:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205077.737890:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737893:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205077.737894:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737895:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737896:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205077.737898:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737901:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205077.737903:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737907:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205077.737909:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737910:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.737910:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.737912:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205077.737914:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205077.737915:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205077.737919:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205077.737921:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205077.737925:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205077.737932:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205077.737936:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205077.737939:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737945:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205077.737946:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737949:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205077.737951:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205077.737953:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205077.737956:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926912:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205077.737959:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205077.737960:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205077.737962:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205077.737963:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205077.737965:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205077.737967:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205077.737969:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205077.737971:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205077.737973:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205077.737975:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205926912, portal 4 00000100:00000001:3.0:1788205077.737976:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205077.737977:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205077.737981:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.737985:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205077.737987:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205077.737988:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205077.737990:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205926912, offset 0 00000400:00000200:3.0:1788205077.737993:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205077.737996:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.737999:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205077.738002:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205077.738004:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205077.738006:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205077.738008:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205077.738011:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59097:59097:256:4294967295] 192.168.201.151@tcp LPNI seq info [59077:59077:8:4294967295] 00000400:00000200:3.0:1788205077.738015:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205077.738021:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205077.738025:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.738028:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205077.738032:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205077.738035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205077.738039:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205077.738042:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205077.738045:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205077.738047:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205077.738050:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205077.738053:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205077.738056:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205077.738057:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205077.738060:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205077.738063:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738071:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.738074:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738085:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.738087:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205077.738090:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205077.738093:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205077.738096:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205077.738098:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301205039 00000100:00000001:3.0:1788205077.738101:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.738102:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301205039 00000100:00000001:3.0:1788205077.738104:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301209089 00000100:00000001:3.0:1788205077.738106:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205077.738108:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.738110:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205077.738114:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205077/real 1788205077] req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205077.738121:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205077.738123:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205077.738124:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205077.738127:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205077.738128:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205077.738130:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205077.738133:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205077.738142:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738147:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205077.738149:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738150:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.738151:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205077.738155:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738161:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205077.738165:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738171:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205077.738173:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738175:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205077.738176:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738178:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205077.738180:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205077.738195:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205077.738199:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205077.738201:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205077.738205:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205077.738212:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205926912:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205077.738215:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738217:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.738220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205926656/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738227:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205077.738228:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205077.738230:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205077.738233:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205077.738234:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205077.738235:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205077.738237:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205077.738239:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788205077.738241:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205077.738243:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205077.738244:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205077.738247:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205077.738248:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738250:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.738253:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205926784/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738259:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205077.738260:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205077.738262:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205077.738264:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205077.738265:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205077.738266:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205077.738268:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205077.738269:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:3.0:1788205077.738272:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205077.738273:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205077.738274:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205077.738276:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205077.738277:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738279:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205077.738283:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205926912/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205077.738289:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205077.738290:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205077.738291:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205077.738294:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205077.738295:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205077.738296:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205077.738298:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205077.738300:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788205077.738301:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205077.738303:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205077.738304:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205077.738306:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205077.738307:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738309:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205077.738313:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205077.738314:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205077.738316:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205077.738318:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205077.928543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205077.928552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205077.928561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205077.928562:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205077.928566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205077.928567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205077.928571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205077.928572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205077.928630:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205077.928634:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205077.928637:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205078.248521:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205078.248529:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205078.248531:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205078.248535:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205078.248539:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205078.248543:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205078.248545:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205078.248547:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205078.248550:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205078.696473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205078.696480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205078.696483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205078.696487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205078.696491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205078.696495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205078.696497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205078.696499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205078.696502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205078.760515:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205078.760516:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205078.760525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205078.760528:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205078.760529:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205078.760532:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205078.760539:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205078.760539:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205078.760546:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205078.760546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205078.760553:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205078.760554:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205078.760558:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205078.760562:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205078.760565:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205078.760568:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205078.760571:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205078.760574:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205078.952488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205078.952498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205078.952506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205078.952508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205078.952513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205078.952514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205078.952518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205078.952519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205078.952584:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205078.952589:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205078.952594:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205079.272516:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205079.272527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205079.272529:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205079.272536:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205079.272540:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205079.272546:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205079.272549:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205079.272552:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205079.272556:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.720497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205079.720505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205079.720507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.720512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205079.720516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205079.720520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.720523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.720526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.720529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.784554:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205079.784562:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205079.784565:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205079.784570:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205079.784573:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205079.784577:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205079.784579:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205079.784582:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205079.784585:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.785232:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205079.785235:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205079.785237:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.785240:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205079.785242:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205079.785245:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.785246:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.785248:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.785250:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205079.976671:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205079.976676:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205079.976679:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205079.976680:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205079.976682:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205079.976683:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205079.976685:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205079.976686:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205079.976745:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205079.976749:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205079.976752:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205080.296517:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205080.296529:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205080.296532:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205080.296539:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205080.296544:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205080.296548:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205080.296560:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205080.296563:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205080.296566:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.744835:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205080.744840:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205080.744841:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205080.744845:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205080.744848:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205080.744851:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205080.744853:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205080.744854:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205080.744856:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205080.808489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205080.808495:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205080.808497:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205080.808501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205080.808503:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205080.808506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205080.808509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205080.808510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205080.808512:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.808587:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205080.808590:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205080.808591:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205080.808594:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205080.808596:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205080.808598:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205080.808599:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205080.808601:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205080.808603:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205081.000491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205081.000498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205081.000503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205081.000504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205081.000506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205081.000507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205081.000508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205081.000509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205081.000557:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205081.000560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205081.000563:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205081.320491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205081.320499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205081.320501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205081.320506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:2.0:1788205081.320509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205081.320513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205081.320516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205081.320518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205081.320521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.768494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205081.768502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205081.768504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205081.768510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205081.768513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205081.768517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205081.768520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205081.768522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205081.768525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205081.832498:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205081.832505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205081.832508:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205081.832514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205081.832517:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205081.832522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205081.832525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205081.832527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205081.832530:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.834205:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205081.834211:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205081.834212:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205081.834215:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205081.834217:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205081.834219:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205081.834221:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205081.834223:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205081.834225:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205082.024538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205082.024547:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205082.024549:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205082.024553:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024556:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205082.024558:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024560:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024562:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205082.024567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205082.024570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205082.024571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205082.024628:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205082.024632:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205082.024635:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205082.344511:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301209089, at 4301209600 00000100:00000010:1.0:1788205082.344527:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4480. 00000100:00000040:0.0:1788205082.346219:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301209089, at 4301209601 00000100:00000010:0.0:1788205082.346228:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f42a0. 00000100:00000001:3.0:1788205082.347034:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205082.347039:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205082.347041:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205082.347045:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205082.347047:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205082.347051:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.347053:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205082.347056:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.347059:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205082.792458:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205082.792465:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205082.792467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205082.792472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205082.792475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205082.792479:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205082.792481:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205082.792484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205082.792487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205082.856611:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205082.856614:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205082.856620:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205082.856622:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205082.856623:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856626:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788205082.856629:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788205082.856631:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205082.856634:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205082.856637:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205082.856639:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205082.856643:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205082.856646:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205082.856647:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6537 00000100:00000001:1.0:1788205082.856651:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205082.856660:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205082.856668:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205082.856672:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205082.856675:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856679:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.856682:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205082.856696:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205082.856700:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205082.856702:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205082.856705:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205082.856707:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205082.856709:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.856711:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205082.856713:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.856715:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856747:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205082.856752:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205082.856755:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205082.856757:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205082.856761:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205082.856763:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205082.856765:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205082.856769:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856771:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.856773:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856775:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205082.856778:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205082.856780:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856783:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205082.856786:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205082.856789:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205082.856792:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856794:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205082.856797:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205082.856806:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205082.856809:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205082.856814:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205082.856815:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856818:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205082.856820:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205082.856824:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205082.856844:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205082.856851:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205082.856861:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205082.856871:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205082.856875:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205082.856880:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205082.856904:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205082.856907:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205082.856910:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205082.856912:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205082.856913:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205082.856916:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205082.856919:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6537 00000400:00000200:2.0:1788205082.856923:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205082.856926:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205082.856928:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205082.856931:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856933:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.856935:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856937:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205082.856939:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205082.856940:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205082.856942:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205082.856944:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205082.856946:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205082.856949:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205082.856952:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856954:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.856956:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856957:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205082.856960:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205082.856963:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205082.856967:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205082.856968:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205082.856974:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205082.856976:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856978:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205082.856982:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205082.856984:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856986:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205082.856990:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205082.856992:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205082.856994:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205082.856995:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.856997:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205082.856998:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205082.857000:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205082.857004:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788205082.857007:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205082.857012:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205082.857020:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205082.857022:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205082.857026:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205082.857035:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205082.857037:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205082.857041:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205082.857041:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205082.857046:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205082.857047:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205082.857049:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205082.857051:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205082.857053:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788205082.857056:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6537 00000400:00000200:2.0:1788205082.857062:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:3.0:1788205082.857065:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205082.857067:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205082.857072:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205082.857078:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205082.857081:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205082.857083:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205082.857084:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205082.857088:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205082.857091:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205082.857092:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205082.857103:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788205082.857108:0:12771:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788205082.857109:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205082.857111:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205082.857116:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1788205082.857116:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205927040:192.168.201.151@tcp:250:kworker.0 00000020:00000040:2.0:1788205082.857119:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:3.0:1788205082.857123:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788205082.857124:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205082.857129:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 02000000:00000001:3.0:1788205082.857129:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205082.857133:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205082.857135:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205082.857137:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205082.857139:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.857143:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000010:3.0:1788205082.857144:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 00000100:00000001:2.0:1788205082.857146:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205082.857148:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205082.857151:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205082.857156:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788205082.857158:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205082.857161:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205082.857164:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1788205082.857165:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000001:2.0:1788205082.857168:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205082.857173:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205082.857173:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:2.0:1788205082.857176:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205082.857180:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205082.857182:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.857184:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205082.857188:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205082.857190:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205082.857193:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205082.857194:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.857195:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205082.857197:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205082.857199:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205082.857203:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788205082.857206:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205082.857211:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205082.857218:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205082.857220:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205082.857224:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205082.857232:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205082.857235:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205082.857238:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205082.857240:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205082.857241:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205082.857244:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205082.857246:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6537 00000400:00000200:2.0:1788205082.857250:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205082.857253:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205082.857255:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205082.857258:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857260:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.857262:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857264:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205082.857266:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205082.857267:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205082.857268:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205082.857270:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205082.857273:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205082.857275:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205082.857278:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857280:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205082.857282:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857284:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205082.857286:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205082.857288:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857290:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205082.857292:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205082.857294:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205082.857296:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.857298:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205082.857300:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205082.857302:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.857304:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205082.857308:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205082.857310:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205082.857313:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205082.857314:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205082.857316:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205082.857317:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205082.857319:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205082.857322:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205082.857325:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205082.857327:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927040, portal 25 00000100:00100000:2.0:1788205082.857331:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205082.857332:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205082.857335:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000001:2.0:1788205082.857345:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788205082.857346:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205082.857350:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.857359:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205082.857360:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205082.857399:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205082.857404:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205082.857405:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000040:2.0:1788205082.857411:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6547) 00000100:00000200:3.0:1788205082.857412:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205927040, offset 0 00000400:00000200:3.0:1788205082.857422:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205082.857431:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.857437:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205082.857441:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205082.857446:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205082.857449:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205082.857452:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205082.857456:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59098:59098:256:4294967295] 192.168.201.151@tcp LPNI seq info [59078:59078:8:4294967295] 00000400:00000200:3.0:1788205082.857460:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.857470:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205082.857476:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.857481:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205082.857487:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205082.857491:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205082.857496:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205082.857499:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205082.857503:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.857506:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205082.857511:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205082.857516:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205082.857522:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205082.857524:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205082.857528:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205082.857533:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857541:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.857545:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857552:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.857554:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205082.857559:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205082.857563:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205082.857566:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301214210 00000100:00000010:3.0:1788205082.857571:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c00. 00000100:00000001:3.0:1788205082.857577:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205082.857593:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301214210 00000100:00000010:3.0:1788205082.857596:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4f60. 00000100:00000001:3.0:1788205082.857599:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205082.857601:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205082.857604:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.857606:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205082.857609:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205082/real 1788205082] req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205082.857618:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205082.857619:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205082.857621:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205082.857623:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205082.857625:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205082.857627:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205082.857630:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205082.857637:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857643:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205082.857644:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857647:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.857649:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205082.857652:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857658:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205082.857662:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857669:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205082.857671:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857673:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.857674:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857676:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205082.857678:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205082.857681:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205082.857684:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205082.857686:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205082.857690:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205082.857697:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205927040:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205082.857701:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205082.857704:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857709:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205082.857711:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.857713:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205082.857715:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205082.857717:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205082.857719:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927168:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205082.857722:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205082.857723:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205082.857725:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.857726:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205082.857729:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205082.857731:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205082.857734:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205082.857736:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205082.857739:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205082.857741:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927168, portal 10 00000100:00000001:3.0:1788205082.857743:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205082.857744:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788205082.857750:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857756:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205082.857758:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205082.857760:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205082.857762:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205927168, offset 0 00000400:00000200:3.0:1788205082.857766:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205082.857769:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.857772:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205082.857775:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205082.857778:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205082.857780:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205082.857782:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205082.857785:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59099:59099:256:4294967295] 192.168.201.151@tcp LPNI seq info [59079:59079:8:4294967295] 00000400:00000200:3.0:1788205082.857789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.857794:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205082.857798:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.857801:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205082.857805:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205082.857807:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205082.857812:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205082.857814:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205082.857818:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.857819:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205082.857822:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205082.857826:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205082.857829:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205082.857830:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205082.857833:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205082.857836:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857844:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.857846:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857853:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.857855:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205082.857858:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205082.857862:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205082.857866:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205082.857868:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301210267 00000100:00000001:3.0:1788205082.857870:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.857872:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301210267 00000100:00000001:3.0:1788205082.857874:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.857876:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205082.857878:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.857879:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205082.857884:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205082/real 1788205082] req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205082.857892:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205082.857893:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205082.857895:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205082.857897:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205082.857899:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205082.857901:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205082.857904:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205082.857913:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857919:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205082.857920:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857922:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.857923:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205082.857927:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857933:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205082.857937:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857943:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205082.857945:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857947:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.857948:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857949:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205082.857951:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205082.857954:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205082.857958:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205082.857960:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205082.857964:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205082.857972:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927168:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205082.857976:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.857978:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.857981:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205927040/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.857987:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205082.857990:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205082.857992:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205082.857995:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205082.857996:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205082.857998:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205082.858000:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205082.858002:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:3.0:1788205082.858006:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205082.858007:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205082.858009:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788205082.858012:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205082.858013:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858015:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.858018:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205927168/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858024:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205082.858025:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205082.858027:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205082.858029:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205082.858030:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205082.858031:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205082.858032:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205082.858033:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788205082.858034:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205082.858035:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205082.858037:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788205082.858038:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205082.858039:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858040:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858046:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205082.858047:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858048:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205082.858049:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205082.858050:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205082.858052:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858056:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205082.858057:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.858059:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205082.858060:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205082.858061:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205082.858063:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205082.858065:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205082.858066:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205082.858067:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.858068:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205082.858070:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205082.858071:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205082.858074:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205082.858075:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205082.858077:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205082.858079:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927296, portal 4 00000100:00000001:3.0:1788205082.858081:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205082.858081:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788205082.858085:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858091:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205082.858120:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205082.858123:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205082.858127:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205927296, offset 0 00000400:00000200:3.0:1788205082.858131:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205082.858135:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.858139:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205082.858142:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205082.858144:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205082.858146:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205082.858148:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205082.858151:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59100:59100:256:4294967295] 192.168.201.151@tcp LPNI seq info [59080:59080:8:4294967295] 00000400:00000200:3.0:1788205082.858155:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.858161:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205082.858165:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.858168:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205082.858172:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205082.858175:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205082.858179:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205082.858181:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205082.858185:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.858186:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205082.858189:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205082.858192:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205082.858195:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205082.858197:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205082.858200:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205082.858203:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858210:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.858213:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858220:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858221:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205082.858224:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205082.858228:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205082.858231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205082.858233:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301210153 00000100:00000001:3.0:1788205082.858235:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.858237:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301210153 00000100:00000001:3.0:1788205082.858239:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.858241:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205082.858243:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858245:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205082.858249:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205082/real 1788205082] req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205082.858256:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205082.858258:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205082.858259:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205082.858261:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205082.858263:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205082.858265:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205082.858268:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205082.858276:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858282:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205082.858284:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858285:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.858287:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205082.858290:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858297:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205082.858300:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858307:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205082.858310:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858311:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.858313:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858314:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205082.858317:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205082.858319:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205082.858322:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205082.858324:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205082.858328:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205082.858336:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205082.858340:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205082.858343:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858349:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205082.858350:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.858352:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205082.858354:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205082.858356:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205082.858359:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927424:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205082.858362:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205082.858382:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205082.858384:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205082.858386:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205082.858389:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205082.858391:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205082.858394:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205082.858397:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205082.858400:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205082.858402:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927424, portal 4 00000100:00000001:3.0:1788205082.858404:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205082.858405:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:3.0:1788205082.858411:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858418:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205082.858420:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205082.858422:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205082.858424:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205927424, offset 0 00000400:00000200:3.0:1788205082.858428:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205082.858432:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.858435:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205082.858438:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205082.858440:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205082.858442:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205082.858444:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205082.858447:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59101:59101:256:4294967295] 192.168.201.151@tcp LPNI seq info [59081:59081:8:4294967295] 00000400:00000200:3.0:1788205082.858451:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205082.858457:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205082.858461:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.858464:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205082.858468:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205082.858470:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205082.858475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205082.858477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205082.858481:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205082.858483:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205082.858485:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205082.858489:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205082.858491:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205082.858493:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205082.858496:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205082.858499:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858507:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.858510:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858516:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858517:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205082.858521:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205082.858523:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205082.858526:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205082.858528:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301210160 00000100:00000001:3.0:1788205082.858531:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.858533:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301210160 00000100:00000001:3.0:1788205082.858535:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301214210 00000100:00000001:3.0:1788205082.858536:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205082.858538:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858540:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205082.858544:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205082/real 1788205082] req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205082.858551:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205082.858553:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205082.858554:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205082.858556:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205082.858558:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205082.858560:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205082.858563:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205082.858570:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858580:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205082.858582:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858583:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.858584:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205082.858588:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858594:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205082.858597:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858603:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205082.858605:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858607:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205082.858608:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858610:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205082.858611:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205082.858614:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205082.858617:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205082.858619:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205082.858623:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205082.858630:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927424:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205082.858633:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858635:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.858638:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205927296/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858644:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205082.858646:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205082.858649:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205082.858651:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205082.858652:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205082.858654:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205082.858655:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205082.858657:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788205082.858659:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205082.858661:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205082.858662:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788205082.858664:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205082.858666:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858667:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205082.858670:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205927424/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205082.858676:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205082.858677:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205082.858679:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205082.858681:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205082.858682:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205082.858683:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205082.858685:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205082.858686:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788205082.858688:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205082.858689:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205082.858690:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788205082.858692:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205082.858693:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858695:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205082.858698:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205082.858699:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205082.858701:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205082.858702:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049183:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205083.049192:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205083.049194:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205083.049197:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049202:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205083.049204:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049206:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049208:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049212:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205083.049214:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049217:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205083.049219:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205083.049220:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205083.049222:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049224:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205083.049225:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049226:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049228:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049231:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205083.049233:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205083.049234:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205083.049236:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049238:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205083.049239:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049240:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205083.049242:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205083.049304:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205083.049308:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205083.049311:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205083.368484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205083.368490:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205083.368492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205083.368495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205083.368498:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205083.368501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205083.368503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205083.368504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205083.368507:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.816480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205083.816486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205083.816488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205083.816495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205083.816498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205083.816502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205083.816505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205083.816507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205083.816510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.880458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205083.880466:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205083.880468:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205083.880474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205083.880476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205083.880479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205083.880481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205083.880483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.880485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205083.880485:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205083.880489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205083.880491:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205083.880492:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205083.880495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205083.880500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205083.880500:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205083.880503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205083.880505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205084.072533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205084.072539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205084.072545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205084.072546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205084.072550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205084.072551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205084.072554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205084.072556:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205084.072609:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205084.072612:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205084.072614:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205084.392493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205084.392500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205084.392502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205084.392507:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205084.392511:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205084.392515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.392518:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205084.392520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.392523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.840506:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205084.840513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205084.840516:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.840521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205084.840525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205084.840528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.840531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.840533:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.840536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.904506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205084.904514:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205084.904516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205084.904522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205084.904525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205084.904528:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.904531:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205084.904533:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.904536:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.905135:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205084.905138:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205084.905139:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.905142:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205084.905144:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205084.905146:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.905148:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.905149:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.905151:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205085.096478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205085.096485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205085.096492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205085.096493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205085.096496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205085.096498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205085.096501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205085.096503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205085.096560:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205085.096565:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205085.096569:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205085.416466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205085.416473:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205085.416474:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205085.416479:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205085.416482:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205085.416485:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205085.416488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205085.416491:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205085.416493:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.864448:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205085.864455:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205085.864457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205085.864462:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205085.864466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205085.864469:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.864471:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205085.864473:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.864475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205085.928525:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205085.928532:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205085.928534:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205085.928539:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205085.928543:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205085.928546:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205085.928549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205085.928551:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205085.928554:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.930122:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205085.930126:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205085.930128:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205085.930132:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205085.930134:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205085.930136:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.930139:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205085.930141:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.930143:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205086.120538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205086.120544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205086.120548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205086.120549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205086.120552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205086.120553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205086.120555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205086.120557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205086.120621:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205086.120625:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205086.120629:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205086.440491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205086.440499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205086.440500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205086.440506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205086.440509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205086.440514:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205086.440516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205086.440518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205086.440521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.888487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205086.888494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205086.888495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205086.888500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205086.888504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205086.888507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.888510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205086.888512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.888514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.952427:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205086.952430:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205086.952435:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205086.952436:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205086.952437:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.952440:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205086.952442:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205086.952444:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205086.952446:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205086.952450:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205086.952450:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205086.952454:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205086.952455:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205086.952456:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.952461:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205086.952461:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.952464:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.952467:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205087.144526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205087.144532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205087.144538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205087.144539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205087.144542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205087.144543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205087.144546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205087.144546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205087.144603:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205087.144607:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205087.144617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205087.464468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205087.464475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205087.464478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.464484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205087.464488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205087.464492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.464496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.464498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.464502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205087.464515:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301214210, at 4301214720 00000100:00000010:0.0:1788205087.464526:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c00. 00000100:00000040:1.0:1788205087.465410:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301214210, at 4301214721 00000100:00000010:1.0:1788205087.465423:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4f60. 00000100:00000001:3.0:1788205087.912463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205087.912470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205087.912472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.912477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205087.912480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205087.912484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.912486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.912488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.912491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205087.976492:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205087.976498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205087.976499:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205087.976504:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205087.976507:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205087.976510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205087.976513:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205087.976515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205087.976517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205087.976581:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205087.976587:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205087.976590:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205087.976593:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205087.976597:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205087.976599:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205087.976602:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205087.976604:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205087.976605:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6542 00000100:00100000:0.0:1788205087.976607:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205087.976610:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205087.976611:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205087.976613:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205087.976615:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205087.976616:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205087.976617:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205087.976626:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205087.976630:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205087.976633:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.976636:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.976638:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.976640:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205087.976644:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205087.976646:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205087.976648:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205087.976652:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205087.976654:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205087.976656:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205087.976658:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.976660:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.976661:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.976663:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205087.976665:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205087.976668:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205087.976671:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205087.976673:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205087.976676:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205087.976679:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.976681:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205087.976683:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205087.976692:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788205087.976695:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205087.976699:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205087.976701:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.976703:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205087.976706:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205087.976710:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205087.976718:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:1.0:1788205087.976724:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205087.976731:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205087.976738:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205087.976741:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205087.976744:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.976780:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205087.976783:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205087.976785:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205087.976789:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.976794:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.976795:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.976798:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205087.976800:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205087.976803:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.976806:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205927552:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205087.976810:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205087.976814:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205087.976816:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.976818:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205087.976822:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.976825:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205087.976829:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205087.976833:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205087.976839:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205087.976845:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927552, portal 25 00000100:00000001:3.0:1788205087.976847:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205087.976849:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000040:3.0:1788205087.976855:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.976865:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205087.976868:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205087.976870:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205087.976873:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205927552, offset 0 00000400:00000200:3.0:1788205087.976887:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205087.976895:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.976900:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205087.976906:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205087.976909:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205087.976911:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205087.976913:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205087.976916:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59102:59102:256:4294967295] 192.168.201.151@tcp LPNI seq info [59082:59082:8:4294967295] 00000400:00000200:3.0:1788205087.976919:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.976926:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205087.976931:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.976934:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205087.976939:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205087.976941:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205087.976945:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205087.976948:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205087.976950:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.976952:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205087.976956:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205087.976959:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205087.976963:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205087.976964:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205087.976968:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205087.976970:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205087.977001:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205087.977003:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205087.977018:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205087.977019:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205087.977021:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205087.977023:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205087.977026:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6542 00000400:00000200:1.0:1788205087.977029:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205087.977031:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205087.977033:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205087.977035:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977036:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977038:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977040:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205087.977042:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205087.977042:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205087.977044:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205087.977045:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205087.977046:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205087.977047:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788205087.977049:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977050:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977051:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977053:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205087.977054:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205087.977056:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977057:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205087.977058:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205087.977061:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205087.977063:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977064:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205087.977067:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205087.977069:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977071:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205087.977073:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788205087.977075:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205087.977076:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205087.977077:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977078:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205087.977079:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205087.977080:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205087.977083:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788205087.977085:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205087.977089:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205087.977094:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205087.977096:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205087.977099:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977122:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.977125:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977133:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977134:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205087.977140:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205087.977142:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205087.977144:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301219329 00000100:00000010:3.0:1788205087.977147:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4300. 00000100:00000001:3.0:1788205087.977151:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205087.977155:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301219329 00000100:00000010:3.0:1788205087.977157:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4cc0. 00000100:00000001:3.0:1788205087.977159:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205087.977161:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205087.977163:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977164:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205087.977167:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205087/real 1788205087] req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205087.977172:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205087.977173:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205087.977175:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205087.977176:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205087.977178:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205087.977179:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205087.977181:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205087.977186:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977190:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205087.977190:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977192:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977193:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205087.977195:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977200:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205087.977202:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977206:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205087.977208:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977209:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977209:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977211:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205087.977212:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205087.977214:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205087.977216:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205087.977217:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205087.977220:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205087.977224:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205927552:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205087.977227:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977228:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.977229:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205927552/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977233:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205087.977235:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205087.977237:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.977239:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205087.977240:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205087.977241:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205087.977242:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205087.977243:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:3.0:1788205087.977245:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205087.977247:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205087.977248:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00000001:3.0:1788205087.977250:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205087.977251:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977252:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977257:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205087.977258:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977259:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205087.977260:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205087.977260:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205087.977262:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977266:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.977267:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.977269:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205087.977270:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205087.977271:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.977272:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927680:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205087.977274:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205087.977275:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205087.977276:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.977277:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205087.977279:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.977280:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205087.977282:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205087.977283:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205087.977285:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205087.977287:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927680, portal 10 00000100:00000001:3.0:1788205087.977288:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205087.977289:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 00000100:00000040:3.0:1788205087.977293:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977296:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205087.977298:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205087.977299:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205087.977301:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205927680, offset 0 00000400:00000200:3.0:1788205087.977303:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205087.977305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.977307:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205087.977309:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205087.977311:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205087.977313:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205087.977314:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205087.977316:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59103:59103:256:4294967295] 192.168.201.151@tcp LPNI seq info [59083:59083:8:4294967295] 00000400:00000200:3.0:1788205087.977318:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.977322:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205087.977324:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.977326:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205087.977329:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205087.977330:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205087.977333:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205087.977336:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205087.977339:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.977341:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205087.977343:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205087.977347:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205087.977349:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205087.977351:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205087.977353:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205087.977357:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977385:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.977387:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977392:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977393:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205087.977397:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205087.977400:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205087.977403:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205087.977404:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301215387 00000100:00000001:3.0:1788205087.977406:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.977407:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301215387 00000100:00000001:3.0:1788205087.977408:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.977410:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205087.977411:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977412:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205087.977414:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205087/real 1788205087] req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205087.977418:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205087.977419:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205087.977420:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205087.977421:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205087.977422:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205087.977423:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205087.977425:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205087.977430:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977434:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205087.977434:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977435:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977436:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205087.977438:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977441:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205087.977443:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977448:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205087.977449:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977450:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977451:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977452:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205087.977453:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205087.977454:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205087.977456:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205087.977457:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205087.977459:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205087.977463:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927680:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205087.977466:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977467:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.977469:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205927680/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977472:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205087.977473:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205087.977474:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.977475:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205087.977476:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205087.977477:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205087.977478:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205087.977479:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:3.0:1788205087.977480:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205087.977481:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205087.977482:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:3.0:1788205087.977483:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205087.977484:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977486:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977487:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205087.977488:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977489:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205087.977490:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977528:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205087.977530:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205087.977532:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205087.977533:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205087.977535:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205087.977538:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205087.977540:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6542 00000400:00000200:1.0:1788205087.977544:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205087.977548:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205087.977550:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205087.977553:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977554:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977556:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205087.977560:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205087.977562:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205087.977563:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205087.977565:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205087.977567:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205087.977569:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205087.977572:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977574:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977576:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977578:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205087.977580:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205087.977583:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977585:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205087.977586:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205087.977589:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205087.977591:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977592:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205087.977595:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205087.977597:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977599:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205087.977603:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000020:00000040:1.0:1788205087.977605:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205087.977607:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205087.977609:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977610:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205087.977612:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205087.977613:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205087.977617:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000100:00000001:1.0:1788205087.977620:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205087.977625:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205087.977637:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205087.977638:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205087.977640:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205087.977646:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205087.977647:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205087.977649:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205087.977650:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205087.977651:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205087.977653:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205087.977654:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6542 00000400:00000200:1.0:1788205087.977657:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205087.977658:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205087.977660:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205087.977661:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977663:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977664:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977665:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205087.977666:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205087.977667:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205087.977668:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205087.977669:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205087.977670:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205087.977671:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205087.977673:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977674:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205087.977675:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977677:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205087.977678:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205087.977680:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205087.977681:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205087.977682:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205087.977684:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205087.977685:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977686:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205087.977688:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205087.977689:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977690:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205087.977692:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788205087.977694:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205087.977697:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205087.977698:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.977699:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205087.977701:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205087.977702:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205087.977706:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000100:00000001:1.0:1788205087.977709:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205087.977713:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205087.977719:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205087.977721:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205087.977724:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205087.977731:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205087.977735:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6552) 00000100:00000001:3.0:1788205087.977784:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205087.977787:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205087.977788:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205087.977793:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977800:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.977801:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.977804:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205087.977805:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205087.977806:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.977808:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205087.977810:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205087.977812:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205087.977813:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.977814:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205087.977816:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.977817:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205087.977819:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205087.977821:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205087.977823:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205087.977825:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927808, portal 4 00000100:00000001:3.0:1788205087.977826:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205087.977827:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:3.0:1788205087.977831:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977834:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205087.977836:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205087.977837:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205087.977838:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205927808, offset 0 00000400:00000200:3.0:1788205087.977841:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205087.977844:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.977846:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205087.977848:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205087.977849:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205087.977851:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205087.977852:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205087.977854:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59104:59104:256:4294967295] 192.168.201.151@tcp LPNI seq info [59084:59084:8:4294967295] 00000400:00000200:3.0:1788205087.977857:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.977861:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205087.977863:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.977865:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205087.977867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205087.977869:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205087.977872:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205087.977873:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205087.977876:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.977878:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205087.977881:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205087.977884:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205087.977887:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205087.977889:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205087.977891:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205087.977894:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977901:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.977904:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977910:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977911:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205087.977914:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205087.977917:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205087.977920:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205087.977922:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301215273 00000100:00000001:3.0:1788205087.977925:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.977927:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301215273 00000100:00000001:3.0:1788205087.977928:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.977930:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205087.977932:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.977934:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205087.977938:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205087/real 1788205087] req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205087.977945:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205087.977947:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205087.977948:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205087.977950:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205087.977952:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205087.977954:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205087.977957:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205087.977965:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977972:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205087.977973:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977975:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977976:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205087.977980:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977986:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205087.977989:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.977993:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205087.977995:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977996:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.977997:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.977999:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205087.978001:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205087.978004:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205087.978017:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205087.978019:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205087.978023:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205087.978030:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205087.978034:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205087.978036:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978043:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205087.978044:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.978046:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205087.978048:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205087.978050:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205087.978053:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927936:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205087.978056:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205087.978058:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205087.978059:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205087.978061:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205087.978065:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205087.978066:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205087.978069:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205087.978071:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205087.978073:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205087.978076:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205927936, portal 4 00000100:00000001:3.0:1788205087.978079:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205087.978080:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 00000100:00000040:3.0:1788205087.978085:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978091:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205087.978093:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205087.978096:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205087.978098:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205927936, offset 0 00000400:00000200:3.0:1788205087.978101:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205087.978105:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.978108:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205087.978111:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205087.978113:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205087.978116:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205087.978118:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205087.978121:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59105:59105:256:4294967295] 192.168.201.151@tcp LPNI seq info [59085:59085:8:4294967295] 00000400:00000200:3.0:1788205087.978125:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205087.978131:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205087.978135:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.978138:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205087.978140:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205087.978142:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205087.978144:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205087.978146:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205087.978148:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205087.978149:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205087.978151:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205087.978153:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205087.978155:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205087.978156:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205087.978158:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205087.978160:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978168:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.978170:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978174:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.978175:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205087.978176:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205087.978178:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205087.978180:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205087.978181:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301215279 00000100:00000001:3.0:1788205087.978183:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.978184:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301215279 00000100:00000001:3.0:1788205087.978186:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301219329 00000100:00000001:3.0:1788205087.978187:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205087.978188:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.978189:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205087.978191:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205087/real 1788205087] req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205087.978195:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205087.978196:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205087.978197:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205087.978198:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205087.978199:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205087.978200:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205087.978202:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205087.978206:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978210:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205087.978211:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978212:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.978213:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205087.978215:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978219:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205087.978221:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978224:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205087.978225:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978226:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205087.978227:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978228:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205087.978229:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205087.978230:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205087.978232:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205087.978233:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205087.978235:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205087.978239:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205927936:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205087.978241:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978242:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.978244:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205927808/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978247:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205087.978248:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205087.978249:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205087.978251:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205087.978251:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205087.978253:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205087.978254:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205087.978255:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:3.0:1788205087.978256:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205087.978257:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205087.978258:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:3.0:1788205087.978259:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205087.978260:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978261:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205087.978263:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205927936/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205087.978266:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205087.978267:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205087.978268:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205087.978269:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205087.978269:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205087.978270:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205087.978271:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205087.978272:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:3.0:1788205087.978273:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205087.978273:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205087.978274:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000001:3.0:1788205087.978275:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205087.978276:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978277:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205087.978279:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205087.978280:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205087.978281:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205087.978282:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205088.168511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205088.168516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205088.168520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205088.168521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205088.168523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205088.168524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205088.168526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205088.168526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205088.168588:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205088.168592:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205088.168596:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205088.488474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205088.488480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205088.488481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205088.488485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205088.488487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205088.488490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205088.488492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205088.488495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205088.488497:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205088.936499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205088.936506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205088.936508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205088.936513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205088.936516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205088.936520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205088.936522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205088.936524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205088.936527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205089.000473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205089.000477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205089.000479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205089.000482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205089.000482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205089.000486:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205089.000487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205089.000491:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205089.000492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205089.000496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205089.000496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205089.000501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205089.000501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205089.000504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205089.000507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205089.000508:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205089.000511:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205089.000513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205089.192480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205089.192485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205089.192489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205089.192490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205089.192492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205089.192493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205089.192495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205089.192497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205089.192558:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205089.192563:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205089.192566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205089.512459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205089.512465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205089.512467:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205089.512472:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205089.512476:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205089.512479:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.512481:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205089.512484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.512486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.960480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205089.960488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205089.960490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205089.960495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205089.960499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205089.960502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.960505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205089.960507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.960510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205090.024498:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205090.024505:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205090.024505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205090.024509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205090.024510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205090.024511:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205090.024515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205090.024516:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205090.024519:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205090.024521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205090.024523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205090.024525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205090.024527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205090.024531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205090.024535:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205090.024535:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205090.024538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205090.024541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205090.216519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205090.216526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205090.216532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205090.216533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205090.216544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205090.216545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205090.216548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205090.216549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205090.216618:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205090.216622:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205090.216625:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205090.536486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205090.536493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205090.536496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205090.536501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205090.536504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205090.536508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205090.536511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205090.536513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205090.536516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205090.984682:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205090.984690:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205090.984692:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205090.984698:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205090.984701:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205090.984706:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205090.984709:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205090.984711:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205090.984714:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205091.048472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205091.048472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205091.048478:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205091.048479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205091.048481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205091.048482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205091.048487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205091.048488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205091.048491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205091.048493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205091.048494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205091.048498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205091.048498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205091.048500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205091.048504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205091.048504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205091.048508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205091.048511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205091.240478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205091.240484:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205091.240487:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205091.240489:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240491:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205091.240493:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240494:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240496:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205091.240499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205091.240502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205091.240502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205091.240570:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205091.240574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205091.240577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205091.560500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205091.560510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205091.560513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205091.560520:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205091.560523:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205091.560528:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205091.560530:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205091.560533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205091.560537:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205092.008497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205092.008505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205092.008507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205092.008512:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205092.008515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205092.008518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.008521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205092.008523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.008526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205092.072519:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205092.072525:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205092.072527:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205092.072531:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205092.072534:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205092.072537:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205092.072539:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205092.072541:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205092.072543:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205092.072613:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205092.072616:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205092.072616:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205092.072618:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205092.072619:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205092.072621:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.072629:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205092.072629:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.072631:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205092.264620:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205092.264628:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205092.264633:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205092.264635:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205092.264638:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205092.264640:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205092.264643:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205092.264644:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205092.264718:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205092.264721:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205092.264724:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205092.584500:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301219329, at 4301219840 00000100:00000010:1.0:1788205092.584516:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4cc0. 00000100:00000040:0.0:1788205092.584612:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301219329, at 4301219840 00000100:00000010:0.0:1788205092.584618:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4300. 00000100:00000001:3.0:1788205092.584704:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205092.584707:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205092.584709:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205092.584713:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205092.584716:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205092.584718:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.584721:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205092.584723:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205092.584725:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.032478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205093.032483:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205093.032485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.032488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205093.032491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205093.032494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.032495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.032497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.032499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205093.096443:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205093.096443:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788205093.096449:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205093.096450:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205093.096453:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096454:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788205093.096458:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205093.096458:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205093.096459:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205093.096461:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205093.096461:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205093.096463:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205093.096465:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205093.096465:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205093.096467:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205093.096469:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205093.096469:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205093.096470:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205093.096473:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205093.096478:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205093.096478:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6547 00000100:00000001:3.0:1788205093.096480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.096483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.096486:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205093.096488:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205093.096494:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205093.096501:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205093.096505:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096508:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205093.096511:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096513:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205093.096517:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205093.096519:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205093.096521:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205093.096524:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205093.096526:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205093.096528:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205093.096531:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096533:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205093.096534:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096535:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205093.096537:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205093.096539:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096542:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205093.096544:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205093.096546:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205093.096548:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096549:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205093.096551:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205093.096558:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788205093.096561:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205093.096563:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205093.096565:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096567:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205093.096569:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205093.096572:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205093.096577:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788205093.096582:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205093.096589:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205093.096596:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205093.096598:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205093.096601:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205093.096614:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205093.096617:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.096621:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205093.096622:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205093.096623:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205093.096626:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205093.096628:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6547 00000400:00000200:1.0:1788205093.096632:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205093.096635:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205093.096637:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205093.096640:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096642:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205093.096644:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096646:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205093.096647:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788205093.096648:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205093.096650:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788205093.096650:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788205093.096651:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205093.096652:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788205093.096653:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205093.096655:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205093.096657:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205093.096657:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205093.096660:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096662:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205093.096664:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205093.096665:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205093.096666:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205093.096666:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205093.096668:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205093.096669:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788205093.096670:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096672:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.096672:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205093.096673:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205093.096675:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.096677:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205928064:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788205093.096678:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205093.096680:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096681:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205093.096681:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:1.0:1788205093.096685:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205093.096685:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205093.096687:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788205093.096688:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096689:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.096689:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1788205093.096692:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 02000000:00000010:3.0:1788205093.096693:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 00000020:00000040:1.0:1788205093.096695:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205093.096695:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205093.096697:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205093.096698:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096699:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788205093.096700:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788205093.096701:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205093.096702:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205093.096704:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 02000000:00000010:1.0:1788205093.096705:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788205093.096708:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205093.096710:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00100000:1.0:1788205093.096712:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205093.096716:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928064, portal 25 00000100:00000001:1.0:1788205093.096718:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205093.096718:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205093.096720:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096720:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:1.0:1788205093.096723:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205093.096725:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205093.096731:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096732:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788205093.096734:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788205093.096735:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788205093.096737:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:3.0:1788205093.096737:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000001:1.0:1788205093.096738:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205093.096740:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788205093.096740:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205928064, offset 0 00000100:00000001:1.0:1788205093.096742:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205093.096744:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6547 00000400:00000200:3.0:1788205093.096746:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205093.096747:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205093.096749:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205093.096751:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205093.096754:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205093.096754:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205093.096756:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205093.096757:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096759:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205093.096759:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:1.0:1788205093.096761:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205093.096762:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:3.0:1788205093.096763:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1788205093.096764:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205093.096765:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205093.096767:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205093.096767:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:1.0:1788205093.096769:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205093.096771:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1788205093.096772:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096773:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205093.096774:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:1.0:1788205093.096775:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096777:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205093.096778:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59106:59106:256:4294967295] 192.168.201.151@tcp LPNI seq info [59086:59086:8:4294967295] 00000020:00000040:1.0:1788205093.096780:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205093.096782:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788205093.096782:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205093.096784:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205093.096785:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205093.096787:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205093.096789:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096791:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205093.096791:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000008:00100000:1.0:1788205093.096793:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205093.096795:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096796:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205093.096797:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000010:1.0:1788205093.096799:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000020:00000040:1.0:1788205093.096800:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205093.096802:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:3.0:1788205093.096802:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 02000000:00000001:1.0:1788205093.096804:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096805:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205093.096806:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205093.096807:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205093.096808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000010:1.0:1788205093.096810:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000800:00000200:3.0:1788205093.096811:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000100:00000001:1.0:1788205093.096813:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205093.096817:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788205093.096817:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205093.096820:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000100:00000001:1.0:1788205093.096823:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1788205093.096824:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096825:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205093.096826:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000040:1.0:1788205093.096828:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205093.096831:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788205093.096835:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205093.096836:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1788205093.096837:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.096840:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788205093.096840:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000001:1.0:1788205093.096841:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788205093.096842:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00080000:1.0:1788205093.096843:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205093.096845:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205093.096845:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1788205093.096847:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6547 00000100:00000200:3.0:1788205093.096849:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205093.096850:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205093.096853:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205093.096855:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205093.096857:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205093.096857:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205093.096859:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205093.096860:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205093.096861:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096863:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205093.096865:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205093.096866:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788205093.096867:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205093.096868:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205093.096869:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205093.096869:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1788205093.096871:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205093.096873:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788205093.096875:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205093.096876:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096878:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205093.096878:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:1.0:1788205093.096879:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096881:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205093.096881:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301224449 00000020:00000040:1.0:1788205093.096883:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000010:3.0:1788205093.096885:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4d20. 00000100:00080000:1.0:1788205093.096886:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205093.096888:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205093.096889:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205093.096891:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205093.096891:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788205093.096893:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.096895:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205093.096896:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301224449 00000008:00100000:1.0:1788205093.096897:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205093.096899:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205093.096899:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f47e0. 00000100:00000001:1.0:1788205093.096900:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096901:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000010:1.0:1788205093.096903:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000100:00000001:3.0:1788205093.096904:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1788205093.096905:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205093.096907:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205093.096907:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205093.096908:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096909:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788205093.096910:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205093.096911:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205093.096912:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.096913:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788205093.096915:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000400:00000001:3.0:1788205093.096936:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205093.096938:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205093.096939:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205093.096942:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205093.096943:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:1.0:1788205093.096946:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096946:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205093.096949:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205093.096950:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205093.096957:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788205093.096958:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205093.096959:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205093.096963:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.096965:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205093.096966:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.096969:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205093.096970:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.096971:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205093.096974:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6558) 00000100:00100000:3.0:1788205093.096974:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.096980:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205093.096985:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.096992:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205093.096994:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.096996:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.096997:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.096999:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205093.097001:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205093.097004:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.097006:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205093.097008:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205093.097010:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.097016:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205928064:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205093.097018:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097019:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097021:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205928064/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097025:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205093.097026:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097028:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205093.097030:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205093.097031:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205093.097032:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205093.097034:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205093.097035:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:3.0:1788205093.097038:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205093.097039:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205093.097040:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:3.0:1788205093.097042:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205093.097042:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097044:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097048:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205093.097049:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097050:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205093.097051:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205093.097052:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205093.097055:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097058:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.097059:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097061:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205093.097062:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205093.097063:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.097064:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928192:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205093.097066:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205093.097067:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205093.097068:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097070:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097072:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205093.097073:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205093.097075:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205093.097077:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205093.097079:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205093.097081:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928192, portal 10 00000100:00000001:3.0:1788205093.097082:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205093.097083:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:3.0:1788205093.097087:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097090:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205093.097092:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205093.097093:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205093.097095:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205928192, offset 0 00000400:00000200:3.0:1788205093.097097:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097099:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097101:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205093.097103:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205093.097105:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205093.097106:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205093.097108:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097110:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59107:59107:256:4294967295] 192.168.201.151@tcp LPNI seq info [59087:59087:8:4294967295] 00000400:00000200:3.0:1788205093.097112:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097116:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205093.097119:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097120:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205093.097123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205093.097125:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788205093.097127:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205093.097129:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1788205093.097131:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097133:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205093.097134:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205093.097136:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205093.097138:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205093.097139:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205093.097140:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205093.097142:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097147:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097148:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097152:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097153:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205093.097154:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205093.097157:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205093.097158:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205093.097160:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301220506 00000100:00000001:3.0:1788205093.097161:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097163:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301220506 00000100:00000001:3.0:1788205093.097164:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097165:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205093.097167:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097168:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205093.097170:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205093.097174:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205093.097175:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205093.097175:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205093.097176:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205093.097177:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205093.097178:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205093.097180:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205093.097185:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097188:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205093.097189:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097190:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097191:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205093.097192:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097196:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205093.097198:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097201:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205093.097202:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097203:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097204:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097205:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205093.097206:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205093.097208:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.097210:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205093.097211:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205093.097213:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.097217:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928192:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205093.097219:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205093.097221:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097224:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.097225:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097226:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205093.097227:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205093.097228:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.097229:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205093.097231:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205093.097231:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205093.097232:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097233:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097235:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205093.097236:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205093.097238:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205093.097239:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205093.097241:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205093.097243:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928320, portal 4 00000100:00000001:3.0:1788205093.097244:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205093.097244:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:3.0:1788205093.097248:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097255:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205093.097256:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205093.097257:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205093.097258:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205928320, offset 0 00000400:00000200:3.0:1788205093.097260:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097262:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097264:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205093.097265:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205093.097266:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205093.097268:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205093.097269:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097271:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59108:59108:256:4294967295] 192.168.201.151@tcp LPNI seq info [59088:59088:8:4294967295] 00000400:00000200:3.0:1788205093.097273:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097276:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205093.097279:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097280:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205093.097282:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205093.097284:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788205093.097286:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205093.097297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1788205093.097299:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097301:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205093.097302:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205093.097304:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205093.097306:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205093.097307:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205093.097309:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205093.097311:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097315:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097317:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097325:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097326:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205093.097327:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205093.097329:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205093.097331:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205093.097332:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301220392 00000100:00000001:3.0:1788205093.097333:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097335:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301220392 00000100:00000001:3.0:1788205093.097336:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097337:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205093.097338:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097339:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205093.097341:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205093.097345:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205093.097346:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205093.097346:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205093.097348:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205093.097348:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205093.097349:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205093.097351:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205093.097355:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097359:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205093.097360:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097361:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097361:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205093.097379:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097383:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205093.097386:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097389:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205093.097391:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097392:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097392:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097393:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205093.097395:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205093.097396:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.097397:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205093.097399:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205093.097401:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.097405:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205093.097407:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205093.097409:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097412:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.097413:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097414:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205093.097415:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205093.097416:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.097417:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928448:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205093.097419:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205093.097420:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205093.097421:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205093.097422:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097424:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205093.097425:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205093.097427:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205093.097428:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205093.097430:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205093.097432:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928448, portal 4 00000100:00000001:3.0:1788205093.097433:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205093.097434:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000040:3.0:1788205093.097437:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097440:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205093.097442:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205093.097443:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205093.097444:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205928448, offset 0 00000400:00000200:3.0:1788205093.097446:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097448:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097450:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205093.097451:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205093.097453:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205093.097454:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205093.097456:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205093.097457:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59109:59109:256:4294967295] 192.168.201.151@tcp LPNI seq info [59089:59089:8:4294967295] 00000400:00000200:3.0:1788205093.097460:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205093.097464:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205093.097466:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097468:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205093.097470:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205093.097471:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788205093.097474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205093.097475:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1788205093.097477:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205093.097479:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205093.097480:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205093.097482:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205093.097484:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205093.097485:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205093.097486:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205093.097488:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097493:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097494:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097498:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097499:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205093.097500:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205093.097502:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205093.097503:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205093.097504:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301220399 00000100:00000001:3.0:1788205093.097506:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097508:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301220399 00000100:00000001:3.0:1788205093.097509:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301224449 00000100:00000001:3.0:1788205093.097511:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205093.097513:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097514:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205093.097517:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205093.097521:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205093.097522:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205093.097523:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205093.097524:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205093.097525:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205093.097527:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205093.097528:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205093.097533:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097536:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205093.097537:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097538:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097539:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205093.097540:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097544:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205093.097546:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097549:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205093.097550:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097551:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205093.097552:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097553:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205093.097554:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205093.097555:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.097557:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205093.097558:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205093.097560:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.097564:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928448:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205093.097566:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097567:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097569:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205928192/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097573:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205093.097575:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097576:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205093.097577:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205093.097578:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205093.097578:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205093.097579:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205093.097580:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:3.0:1788205093.097582:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205093.097583:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205093.097584:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:3.0:1788205093.097586:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205093.097586:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097587:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097589:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205928320/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097593:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205093.097593:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097594:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205093.097595:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205093.097596:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205093.097596:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205093.097597:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205093.097598:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:3.0:1788205093.097599:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205093.097600:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205093.097600:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:3.0:1788205093.097601:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205093.097602:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097603:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205093.097604:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205928448/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205093.097607:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205093.097609:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205093.097609:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205093.097610:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205093.097611:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205093.097612:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205093.097613:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205093.097614:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:3.0:1788205093.097615:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205093.097615:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205093.097616:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:3.0:1788205093.097617:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205093.097618:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097619:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.097621:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205093.097622:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205093.097623:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205093.097623:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205093.288479:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205093.288488:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205093.288491:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288497:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205093.288499:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288501:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288504:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205093.288510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205093.288514:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205093.288516:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205093.288517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288519:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205093.288520:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288523:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205093.288528:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205093.288528:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205093.288529:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288530:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205093.288531:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288532:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205093.288533:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205093.288595:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205093.288598:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205093.288601:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205093.608457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205093.608462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205093.608464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205093.608467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205093.608470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205093.608473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.608475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205093.608477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205093.608479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205094.056466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205094.056471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205094.056473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205094.056476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205094.056479:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205094.056482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205094.056484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205094.056486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205094.056488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205094.120475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205094.120475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205094.120482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205094.120482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205094.120484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205094.120484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205094.120488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205094.120488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205094.120492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205094.120492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205094.120496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205094.120496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205094.120499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205094.120501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205094.120503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205094.120504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205094.120505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205094.120507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205094.312500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205094.312505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205094.312511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205094.312512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205094.312516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205094.312517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205094.312520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205094.312521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205094.312588:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205094.312592:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205094.312595:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205094.632847:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205094.632851:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205094.632853:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205094.632857:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205094.632859:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205094.632862:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205094.632863:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205094.632865:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205094.632867:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205095.080538:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205095.080544:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205095.080546:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205095.080551:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205095.080554:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205095.080558:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205095.080561:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205095.080563:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205095.080566:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205095.144438:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205095.144444:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205095.144446:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205095.144451:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205095.144455:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205095.144458:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205095.144460:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205095.144462:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205095.144464:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205095.144843:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205095.144847:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205095.144848:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205095.144851:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205095.144853:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205095.144855:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205095.144857:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205095.144859:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205095.144861:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205095.338033:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205095.338038:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205095.338042:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205095.338043:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205095.338045:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205095.338046:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205095.338048:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205095.338049:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205095.338127:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205095.338134:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205095.338137:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205095.656491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205095.656498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205095.656501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205095.656506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205095.656509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205095.656513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205095.656516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205095.656518:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205095.656521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205096.104598:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205096.104608:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205096.104610:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205096.104618:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205096.104621:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205096.104628:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205096.104630:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205096.104633:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205096.104635:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205096.168522:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205096.168531:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205096.168534:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205096.168540:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205096.168545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205096.168550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205096.168554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205096.168558:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205096.168563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205096.168573:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205096.168581:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205096.168583:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205096.168588:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205096.168592:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205096.168597:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205096.168601:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205096.168605:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205096.168608:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205096.360544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205096.360553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205096.360561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205096.360563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205096.360566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205096.360568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205096.360571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205096.360572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205096.360647:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205096.360653:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205096.360657:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205096.680486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205096.680493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205096.680496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205096.680500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205096.680504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205096.680508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205096.680510:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205096.680513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205096.680516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205097.128481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205097.128487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205097.128489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205097.128494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205097.128497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205097.128501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205097.128503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205097.128505:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205097.128508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205097.192516:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205097.192517:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205097.192526:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205097.192528:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205097.192530:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205097.192533:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205097.192536:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205097.192541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205097.192541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205097.192547:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205097.192549:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205097.192551:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205097.192554:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205097.192556:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205097.192557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205097.192561:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205097.192566:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205097.192569:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205097.384513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205097.384522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205097.384530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205097.384532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205097.384536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205097.384537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205097.384541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205097.384543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205097.384604:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205097.384609:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205097.384613:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205097.704506:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301224449, at 4301224960 00000100:00000040:0.0:1788205097.704516:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301224449, at 4301224960 00000100:00000010:1.0:1788205097.704527:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f47e0. 00000100:00000010:0.0:1788205097.704533:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4d20. 00000100:00000001:3.0:1788205097.704737:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205097.704742:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205097.704745:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205097.704749:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205097.704752:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205097.704756:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205097.704759:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205097.704762:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205097.704765:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.152495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205098.152506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205098.152509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.152516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205098.152520:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205098.152525:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.152528:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.152532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.152536:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.216506:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205098.216516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205098.216518:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.216525:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205098.216530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205098.216534:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.216537:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.216540:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.216543:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.216776:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205098.216779:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205098.216780:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205098.216782:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205098.216784:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205098.216786:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205098.216787:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205098.216788:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205098.216790:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205098.218425:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.218433:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205098.218436:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205098.218440:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205098.218445:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205098.218454:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6553 00000400:00000200:0.0:1788205098.218464:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205098.218471:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205098.218475:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205098.218478:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218481:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218484:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218486:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205098.218491:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205098.218493:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205098.218496:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205098.218500:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205098.218502:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205098.218505:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205098.218508:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218511:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218512:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218515:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205098.218518:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205098.218522:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218525:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205098.218528:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205098.218532:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205098.218535:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218537:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205098.218540:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205098.218552:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205098.218557:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205098.218561:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205098.218563:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218567:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205098.218570:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205098.218575:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205098.218587:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788205098.218596:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205098.218607:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205098.218618:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205098.218623:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205098.218629:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205098.218659:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205098.218662:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.218666:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205098.218667:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205098.218670:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205098.218673:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205098.218676:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6553 00000400:00000200:0.0:1788205098.218680:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205098.218683:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205098.218686:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205098.218688:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218690:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218692:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218694:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205098.218696:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205098.218698:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205098.218699:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205098.218702:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205098.218704:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205098.218706:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205098.218709:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218711:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218713:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218715:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205098.218717:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205098.218720:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218722:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205098.218724:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205098.218730:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205098.218732:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218734:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205098.218739:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205098.218743:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218744:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205098.218748:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205098.218751:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205098.218753:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205098.218755:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218757:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205098.218758:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205098.218760:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205098.218764:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788205098.218767:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205098.218772:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205098.218779:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205098.218781:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205098.218785:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205098.218792:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205098.218795:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.218798:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205098.218800:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205098.218801:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205098.218804:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205098.218807:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6553 00000400:00000200:0.0:1788205098.218810:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205098.218813:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205098.218815:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205098.218818:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218820:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218822:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218824:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205098.218826:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205098.218838:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205098.218840:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205098.218842:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205098.218844:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205098.218846:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205098.218850:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218852:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218854:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218856:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205098.218858:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205098.218860:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218863:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205098.218865:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205098.218867:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205098.218870:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218871:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205098.218874:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205098.218876:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218878:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205098.218881:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205098.218884:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205098.218887:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205098.218888:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.218890:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205098.218891:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205098.218893:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205098.218897:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:0.0:1788205098.218900:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205098.218905:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205098.218912:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205098.218914:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205098.218918:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205098.218925:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205098.218928:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.218932:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205098.218934:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205098.218935:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205098.218938:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205098.218940:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6553 00000400:00000200:0.0:1788205098.218944:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205098.218947:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205098.218949:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205098.218951:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218954:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.218956:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218958:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205098.218960:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205098.218961:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205098.218962:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205098.218964:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205098.218966:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205098.218969:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205098.218971:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.218973:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205098.221089:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205098.221095:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205098.221099:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205098.221103:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205098.221106:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205098.221109:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205098.221114:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205098.221116:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.221119:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205098.221122:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205098.221125:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.221127:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205098.221134:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205098.221137:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205098.221141:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205098.221142:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205098.221144:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205098.221145:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205098.221149:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205098.221154:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788205098.221158:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205098.221165:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205098.221173:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205098.221176:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205098.221179:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205098.221187:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205098.221193:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6563) 00000100:00000001:3.0:1788205098.221400:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205098.221405:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205098.221408:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205098.221416:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221425:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.221428:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.221432:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205098.221434:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205098.221437:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.221441:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205928576:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205098.221445:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205098.221450:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205098.221453:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.221456:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205098.221461:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205098.221463:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205098.221471:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205098.221475:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205098.221483:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205098.221491:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928576, portal 25 00000100:00000001:3.0:1788205098.221493:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205098.221495:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205098.221501:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221510:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205098.221513:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205098.221516:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205098.221520:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205928576, offset 0 00000400:00000200:3.0:1788205098.221527:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205098.221537:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.221544:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205098.221551:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205098.221555:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205098.221560:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205098.221562:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205098.221566:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59110:59110:256:4294967295] 192.168.201.151@tcp LPNI seq info [59090:59090:8:4294967295] 00000400:00000200:3.0:1788205098.221571:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.221582:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205098.221589:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.221595:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205098.221602:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205098.221606:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788205098.221613:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205098.221617:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788205098.221621:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.221624:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205098.221631:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205098.221636:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205098.221643:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205098.221645:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205098.221650:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205098.221655:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221665:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.221669:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221683:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.221685:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205098.221692:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205098.221696:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205098.221700:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301229574 00000100:00000010:3.0:1788205098.221706:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f43c0. 00000100:00000001:3.0:1788205098.221714:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205098.221720:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301229574 00000100:00000010:3.0:1788205098.221723:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4360. 00000100:00000001:3.0:1788205098.221725:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205098.221728:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205098.221732:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.221734:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205098.221739:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205098.221747:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205098.221749:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205098.221751:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205098.221754:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205098.221756:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205098.221759:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205098.221762:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205098.221769:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221776:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205098.221777:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.221779:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.221782:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205098.221785:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221792:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205098.221796:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221803:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205098.221806:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.221808:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.221809:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.221812:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205098.221814:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205098.221817:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.221821:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205098.221824:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205098.221857:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.221865:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205928576:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205098.221870:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205098.221874:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221880:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.221882:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.221885:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205098.221887:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205098.221888:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.221891:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928704:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205098.221894:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205098.221896:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205098.221897:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.221900:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205098.221903:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205098.221905:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205098.221909:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205098.221912:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205098.221916:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205098.221919:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928704, portal 10 00000100:00000001:3.0:1788205098.221920:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205098.221922:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205098.221929:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.221946:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205098.221948:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205098.221951:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205098.221954:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205928704, offset 0 00000400:00000200:3.0:1788205098.221958:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205098.221962:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.221965:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205098.221968:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205098.221970:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205098.221973:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205098.221975:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205098.222180:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59111:59111:256:4294967295] 192.168.201.151@tcp LPNI seq info [59091:59091:8:4294967295] 00000400:00000200:3.0:1788205098.222185:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.222191:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205098.222196:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222199:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205098.222204:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205098.222206:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788205098.222210:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205098.222212:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788205098.222216:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222218:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205098.222221:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205098.222225:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205098.222228:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205098.222230:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205098.222232:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205098.222236:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222244:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.222247:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222253:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222254:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205098.222257:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205098.222262:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205098.222265:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205098.222267:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301225631 00000100:00000001:3.0:1788205098.222269:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222272:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301225631 00000100:00000001:3.0:1788205098.222274:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222276:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205098.222278:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222280:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205098.222284:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205098.222293:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205098.222294:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205098.222296:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205098.222298:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205098.222299:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205098.222301:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205098.222305:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205098.222314:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222321:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205098.222322:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222324:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222326:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205098.222329:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222335:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205098.222339:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222345:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205098.222348:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222349:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222351:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222352:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205098.222355:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205098.222358:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.222361:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205098.222388:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205098.222393:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.222401:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928704:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205098.222405:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205098.222409:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222422:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.222423:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.222426:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205098.222427:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205098.222429:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.222432:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205098.222435:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205098.222437:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205098.222439:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.222441:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205098.222444:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205098.222446:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205098.222449:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205098.222452:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205098.222456:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205098.222459:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928832, portal 4 00000100:00000001:3.0:1788205098.222460:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205098.222462:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205098.222468:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222473:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205098.222475:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205098.222477:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205098.222479:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205928832, offset 0 00000400:00000200:3.0:1788205098.222483:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205098.222487:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.222490:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205098.222493:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205098.222496:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205098.222498:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205098.222500:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205098.222503:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59112:59112:256:4294967295] 192.168.201.151@tcp LPNI seq info [59092:59092:8:4294967295] 00000400:00000200:3.0:1788205098.222507:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.222513:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205098.222517:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222520:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205098.222524:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205098.222526:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788205098.222530:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205098.222533:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788205098.222538:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222539:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205098.222542:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205098.222545:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205098.222548:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205098.222550:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205098.222552:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205098.222556:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222563:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.222566:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222572:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222573:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205098.222576:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205098.222579:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205098.222583:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205098.222585:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301225518 00000100:00000001:3.0:1788205098.222587:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222589:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301225518 00000100:00000001:3.0:1788205098.222591:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222594:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205098.222597:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222598:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205098.222603:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205098.222611:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205098.222612:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205098.222614:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205098.222617:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205098.222618:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205098.222620:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205098.222623:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205098.222632:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222639:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205098.222640:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222642:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222643:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205098.222647:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222654:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205098.222657:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222664:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205098.222666:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222668:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222670:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222672:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205098.222674:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205098.222676:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.222680:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205098.222682:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205098.222686:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.222694:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205098.222698:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205098.222702:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222708:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.222709:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.222712:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205098.222713:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205098.222715:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.222718:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205098.222721:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205098.222723:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205098.222725:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205098.222726:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205098.222730:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205098.222732:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205098.222736:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205098.222739:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205098.222741:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205098.222745:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205928960, portal 4 00000100:00000001:3.0:1788205098.222746:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205098.222748:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788205098.222753:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222760:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205098.222763:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205098.222765:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205098.222767:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205928960, offset 0 00000400:00000200:3.0:1788205098.222771:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205098.222775:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.222779:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205098.222782:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205098.222784:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205098.222787:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205098.222789:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205098.222792:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59113:59113:256:4294967295] 192.168.201.151@tcp LPNI seq info [59093:59093:8:4294967295] 00000400:00000200:3.0:1788205098.222796:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205098.222803:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205098.222807:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222811:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205098.222814:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205098.222818:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788205098.222822:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205098.222824:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1788205098.222842:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205098.222843:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205098.222846:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205098.222849:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205098.222852:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205098.222854:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205098.222857:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205098.222860:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222868:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.222871:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222877:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222878:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205098.222882:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205098.222885:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205098.222889:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205098.222891:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301225524 00000100:00000001:3.0:1788205098.222893:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222895:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301225524 00000100:00000001:3.0:1788205098.222898:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301229574 00000100:00000001:3.0:1788205098.222899:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205098.222901:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.222903:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205098.222907:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205098.222915:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205098.222916:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205098.222918:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205098.222921:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205098.222922:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205098.222924:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205098.222927:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205098.222936:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222942:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205098.222944:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222945:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222947:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205098.222950:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222957:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205098.222960:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.222967:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205098.222970:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222971:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205098.222973:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.222974:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205098.224948:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205098.224954:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.224960:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205098.224963:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205098.224970:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.224980:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205928960:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205098.224985:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.224988:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.224992:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205928576/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.224999:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205098.225002:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205098.225006:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22000. 02000000:00000001:3.0:1788205098.225010:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205098.225013:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205098.225015:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205098.225017:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205098.225020:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:3.0:1788205098.225025:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205098.225027:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205098.225029:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205098.225033:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205098.225034:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.225036:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.225040:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205928704/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.225047:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205098.225049:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205098.225050:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c23c00. 02000000:00000001:3.0:1788205098.225052:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205098.225053:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205098.225054:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205098.225056:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205098.225057:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788205098.225059:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205098.225060:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205098.225062:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205098.225064:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205098.225065:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.225067:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.225070:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205928832/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.225077:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205098.225078:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205098.225079:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22400. 02000000:00000001:3.0:1788205098.225081:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205098.225082:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205098.225083:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205098.225095:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205098.225096:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:3.0:1788205098.225102:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205098.225104:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205098.225105:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205098.225107:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205098.225108:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.225110:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205098.225113:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205928960/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205098.225125:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205098.225127:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205098.225128:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205098.225131:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205098.225132:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205098.225133:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205098.225134:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205098.225136:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788205098.225137:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205098.225138:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205098.225139:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205098.225141:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205098.225142:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.225145:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.225154:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205098.225156:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205098.225158:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205098.225160:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205098.408566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205098.408581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205098.408589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205098.408591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205098.408594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205098.408596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205098.408599:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205098.408601:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205098.408666:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205098.408672:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205098.408677:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205098.728703:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205098.728710:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205098.728712:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205098.728718:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205098.728722:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205098.728727:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.728729:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205098.728732:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205098.728735:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.176462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205099.176469:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205099.176471:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.176476:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205099.176480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205099.176483:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.176486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.176488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.176491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205099.240482:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205099.240482:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205099.240488:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205099.240488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205099.240491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.240491:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205099.240495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205099.240495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205099.240499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205099.240499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205099.240503:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.240503:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205099.240506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205099.240508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205099.240511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.240511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.240512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.240514:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205099.432495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205099.432502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205099.432507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205099.432508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205099.432512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205099.432513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205099.432516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205099.432518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205099.432583:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205099.432587:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205099.432591:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205099.752478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205099.752486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205099.752488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.752492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205099.752495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205099.752498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.752501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.752503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.752505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205100.200451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205100.200459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205100.200461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205100.200466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205100.200469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205100.200473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.200476:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205100.200478:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.200481:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205100.264472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205100.264480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205100.264482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205100.264487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205100.264490:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205100.264493:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.264497:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205100.264500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.264503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205100.264877:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205100.264881:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205100.264882:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205100.264885:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205100.264887:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205100.264890:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205100.264892:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205100.264893:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205100.264895:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205100.456600:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205100.456607:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205100.456612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205100.456614:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205100.456617:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205100.456618:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205100.456621:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205100.456622:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205100.456699:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205100.456702:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205100.456705:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205100.777903:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205100.777911:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205100.777914:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205100.777919:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205100.777923:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205100.777927:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.777929:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205100.777932:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205100.777934:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205101.224497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205101.224504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205101.224506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205101.224511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205101.224515:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205101.224518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.224521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205101.224523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.224526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205101.288487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205101.288494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205101.288496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205101.288501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205101.288504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205101.288508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.288510:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205101.288512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.288515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205101.288606:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205101.288609:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205101.288610:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205101.288613:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205101.288615:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205101.288616:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205101.288618:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205101.288619:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205101.288621:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480858:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205101.480864:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480870:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205101.480873:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205101.480876:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205101.480879:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480881:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205101.480882:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480884:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480885:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480888:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205101.480889:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205101.480891:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205101.480892:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205101.481172:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205101.481175:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205101.481178:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205101.800606:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205101.800613:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205101.800615:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205101.800620:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205101.800623:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205101.800627:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.800629:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205101.800632:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205101.800634:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205102.248472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205102.248480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205102.248482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205102.248487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205102.248490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205102.248494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205102.248497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205102.248499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205102.248502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205102.312478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205102.312479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205102.312485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205102.312485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205102.312487:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205102.312487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205102.312492:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205102.312492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205102.312495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205102.312495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205102.312498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205102.312499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205102.312501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205102.312503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205102.312506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205102.312507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205102.312508:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205102.312510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205102.504659:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205102.504669:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205102.504675:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205102.504676:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205102.504680:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205102.504681:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205102.504684:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205102.504685:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205102.504752:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205102.504781:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205102.504784:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205102.824467:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301229574, at 4301230080 00000100:00000001:3.0:1788205102.824467:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205102.824474:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205102.824477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205102.824479:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4360. 00000100:00100000:3.0:1788205102.824481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205102.824486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205102.824489:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205102.824492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205102.824495:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205102.824499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205102.824595:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301229574, at 4301230080 00000100:00000010:0.0:1788205102.824601:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f43c0. 00000100:00000001:3.0:1788205103.272777:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205103.272784:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205103.272786:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205103.272790:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205103.272794:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205103.272798:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.272800:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205103.272802:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.272805:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.336479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205103.336479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788205103.336482:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205103.336487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205103.336487:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205103.336490:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205103.336492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.336492:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336494:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788205103.336499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205103.336499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:2.0:1788205103.336500:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205103.336506:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205103.336506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205103.336507:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205103.336510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.336511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.336515:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205103.336516:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6558 00000100:00000001:0.0:1788205103.336519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.336524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.336524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1788205103.336526:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205103.336529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205103.336533:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205103.336534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205103.336536:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205103.336541:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336544:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.336547:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336549:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205103.336553:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205103.336556:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205103.336559:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205103.336562:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205103.336565:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205103.336567:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205103.336570:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336572:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.336574:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336576:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205103.336579:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205103.336583:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336585:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205103.336588:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205103.336591:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205103.336594:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336596:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205103.336598:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205103.336607:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205103.336610:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205103.336614:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205103.336616:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336619:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205103.336621:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205103.336625:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205103.336631:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788205103.336636:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205103.336643:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205103.336650:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205103.336653:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.336656:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.336715:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205103.336719:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205103.336720:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205103.336726:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.336734:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205103.336735:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205103.336738:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205103.336751:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205103.336754:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205103.336757:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205929088:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205103.336762:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205103.336765:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205103.336767:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205103.336769:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205103.336772:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205103.336774:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205103.336777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205103.336780:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205103.336785:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205103.336790:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929088, portal 25 00000100:00000001:3.0:1788205103.336792:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205103.336793:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788205103.336797:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.336805:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205103.336807:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205103.336809:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205103.336811:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205929088, offset 0 00000400:00000200:3.0:1788205103.336815:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205103.336821:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205103.336826:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205103.336830:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205103.336833:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205103.336835:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205103.336837:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205103.336840:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59114:59114:256:4294967295] 192.168.201.151@tcp LPNI seq info [59094:59094:8:4294967295] 00000400:00000200:3.0:1788205103.336842:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205103.336848:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205103.336852:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205103.336856:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205103.336861:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205103.336863:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788205103.336867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205103.336869:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1788205103.336872:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205103.336874:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205103.336878:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205103.336881:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205103.336885:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205103.336887:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205103.336890:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205103.336892:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205103.336910:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205103.336913:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205103.336915:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205103.336916:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205103.336917:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205103.336918:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205103.336920:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6558 00000400:00000200:2.0:1788205103.336923:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205103.336924:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205103.336926:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205103.336928:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336929:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.336930:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336932:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205103.336933:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205103.336934:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205103.336935:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205103.336937:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205103.336938:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205103.336939:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205103.336940:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336941:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.336942:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336943:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205103.336945:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205103.336946:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205103.336954:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205103.336956:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205103.336964:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205103.336965:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336966:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205103.336970:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205103.336972:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336973:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205103.336977:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205103.336979:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205103.336981:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205103.336982:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.336984:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205103.336985:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205103.336987:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205103.336991:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205103.336994:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205103.336999:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205103.337006:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205103.337009:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.337012:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337334:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205103.337340:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337348:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337349:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205103.337356:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205103.337361:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205103.337390:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301234690 00000100:00000010:3.0:1788205103.337396:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4720. 00000100:00000001:3.0:1788205103.337402:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205103.337408:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301234690 00000100:00000010:3.0:1788205103.337411:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4420. 00000100:00000001:3.0:1788205103.337414:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205103.337417:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205103.337420:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337422:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205103.337426:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205103/real 1788205103] req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205103.337434:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205103.337436:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205103.337438:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205103.337440:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205103.337442:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205103.337444:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205103.337447:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205103.337456:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337463:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205103.337464:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337466:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205103.337468:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205103.337472:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337479:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205103.337483:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337490:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205103.337492:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337494:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205103.337495:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337498:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205103.337499:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205103.337502:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205103.337506:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205103.337508:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205103.337512:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205103.337520:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205929088:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205103.337524:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337526:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205103.337529:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205929088/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337535:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205103.337537:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205103.337539:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205103.337542:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205103.337544:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205103.337545:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205103.337548:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205103.337550:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:3.0:1788205103.337553:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205103.337555:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205103.337557:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788205103.337560:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205103.337561:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337563:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337569:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205103.337571:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337573:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205103.337574:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205103.337575:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205103.337579:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337586:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205103.337588:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205103.337590:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205103.337591:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205103.337593:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205103.337596:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929216:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205103.337599:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205103.337601:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205103.337602:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205103.337604:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205103.337607:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205103.337609:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205103.337613:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205103.337615:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205103.337619:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205103.337622:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929216, portal 10 00000100:00000001:3.0:1788205103.337623:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205103.337625:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788205103.337631:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337635:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205103.337636:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205103.337638:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205103.337640:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205929216, offset 0 00000400:00000200:3.0:1788205103.337642:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205103.337645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205103.337648:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205103.337650:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205103.337651:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205103.337653:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205103.337654:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205103.337656:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59115:59115:256:4294967295] 192.168.201.151@tcp LPNI seq info [59095:59095:8:4294967295] 00000400:00000200:3.0:1788205103.337659:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205103.337663:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205103.337763:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205103.337767:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205103.337771:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205103.337772:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788205103.337775:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205103.337777:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1788205103.337779:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205103.337780:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205103.337782:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205103.337785:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205103.337787:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205103.337788:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205103.337790:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205103.337792:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337798:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205103.337799:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337803:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337804:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205103.337806:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205103.337809:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205103.337811:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205103.337812:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301230747 00000100:00000001:3.0:1788205103.337814:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301234690 00000100:00000001:3.0:1788205103.337815:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301230747 00000100:00000001:3.0:1788205103.337817:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301234690 00000100:00000001:3.0:1788205103.337818:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205103.337819:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337821:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205103.337823:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205103/real 1788205103] req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205103.337827:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205103.337828:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205103.337829:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205103.337830:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205103.337831:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205103.337832:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205103.337834:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205103.337839:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337842:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205103.337843:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337844:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205103.337845:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205103.337847:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337850:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205103.337852:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337856:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205103.337857:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337858:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205103.337859:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337860:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205103.337861:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205103.337863:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205103.337865:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205103.337866:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205103.337869:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205103.337873:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929216:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205103.337875:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337876:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205103.337877:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205929216/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205103.337881:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205103.337881:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205103.337883:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20c00. 02000000:00000001:3.0:1788205103.337884:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205103.337885:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205103.337885:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205103.337887:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205103.337888:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788205103.337889:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205103.337890:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205103.337891:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788205103.337892:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205103.337893:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337894:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.337896:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205103.337897:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.337898:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205103.337899:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.338100:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205103.338159:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205103.338160:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205103.338229:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205103.338234:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205103.338235:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205103.338237:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205103.338240:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205103.338243:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6558 00000400:00000200:2.0:1788205103.338246:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205103.338249:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205103.338252:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205103.338255:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338257:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.338259:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338260:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205103.338263:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205103.338264:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205103.338266:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205103.338268:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205103.338270:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205103.338272:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205103.338275:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338277:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205103.338279:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338281:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205103.338283:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205103.338285:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338287:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205103.338288:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205103.338290:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205103.338292:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.338293:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205103.338295:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205103.338296:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.338297:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205103.338300:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205103.338301:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205103.338303:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205103.338304:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.338305:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205103.338306:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205103.338308:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205103.338310:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205103.338313:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205103.338316:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205103.338323:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205103.338326:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.338329:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205103.338347:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205103.338350:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205103.338353:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205103.338355:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205103.338356:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205103.338359:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205103.338361:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6558 00000100:00000001:0.0:1788205103.338378:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205103.338381:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205103.338383:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205103.338389:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205103.338395:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205103.338402:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205103.338405:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205103.338406:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205103.338408:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.338411:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205103.338413:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205103.338416:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205103.338417:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205103.338421:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.338421:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338426:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:0.0:1788205103.338427:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929344:192.168.201.151@tcp:8:kworker.0 00000020:00000001:2.0:1788205103.338431:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205103.338433:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:0.0:1788205103.338435:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788205103.338437:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788205103.338438:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1788205103.338441:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 02000000:00000001:0.0:1788205103.338442:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205103.338446:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788205103.338446:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205103.338450:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 02000000:00000010:0.0:1788205103.338452:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205103.338457:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.338457:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338462:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205103.338464:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788205103.338467:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:0.0:1788205103.338470:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000100:00000001:2.0:1788205103.338471:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205103.338476:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000010:0.0:1788205103.338477:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00080000:2.0:1788205103.338481:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000200:0.0:1788205103.338483:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929344, portal 4 00000100:00000001:2.0:1788205103.338486:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.338487:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788205103.338489:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205103.338490:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 02000000:00000001:2.0:1788205103.338494:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205103.338499:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205103.338500:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205103.338503:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205103.338506:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205103.338511:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.338513:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205103.338514:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205103.338519:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000010:2.0:1788205103.338521:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000400:00000010:0.0:1788205103.338523:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000020:00000040:2.0:1788205103.338524:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205103.338528:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1788205103.338529:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205929344, offset 0 02000000:00000001:2.0:1788205103.338529:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205103.338532:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205103.338534:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205103.338536:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205103.338537:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000010:2.0:1788205103.338541:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000400:00000200:0.0:1788205103.338543:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205103.338547:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205103.338550:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205103.338555:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:2.0:1788205103.338556:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205103.338560:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205103.338565:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205103.338570:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205103.338571:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1788205103.338576:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59116:59116:256:4294967295] 192.168.201.151@tcp LPNI seq info [59096:59096:8:4294967295] 00000100:00000001:2.0:1788205103.338576:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205103.338583:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205103.338584:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205103.338597:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205103.338601:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205103.338605:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788205103.338608:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6568) 00000800:00000010:0.0:1788205103.338610:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205103.338616:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205103.338619:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788205103.338625:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205103.338629:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788205103.338633:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205103.338635:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205103.338639:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205103.338642:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205103.338646:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205103.338648:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205103.338650:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205103.338653:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338661:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205103.338664:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338671:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.338672:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205103.338676:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205103.338679:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205103.338682:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205103.338684:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301230634 00000100:00000001:0.0:1788205103.338686:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301234690 00000100:00000001:0.0:1788205103.338689:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301230634 00000100:00000001:0.0:1788205103.338691:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301234690 00000100:00000001:0.0:1788205103.338692:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205103.338694:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.338696:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205103.338700:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205103/real 1788205103] req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205103.338708:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205103.338709:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205103.338711:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205103.338713:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205103.338715:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205103.338717:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205103.338720:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205103.338728:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338752:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205103.338754:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338756:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205103.338757:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205103.338761:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338768:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205103.338771:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338778:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205103.338781:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338782:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205103.338784:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338785:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205103.338787:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205103.338789:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205103.338792:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205103.338795:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205103.338799:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205103.338806:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929344:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205103.338810:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338812:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205103.338815:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205929344/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338821:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205103.338823:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205103.338825:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205103.338828:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205103.338829:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205103.338830:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205103.338832:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205103.338834:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205103.338836:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205103.338837:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205103.338839:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205103.338849:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205103.338851:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338853:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.338856:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205103.338857:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.338859:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205103.338860:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205103.338862:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205103.338865:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338872:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205103.338874:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205103.338876:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205103.338877:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205103.338879:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205103.338882:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929472:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205103.338885:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205103.338887:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205103.338888:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205103.338890:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205103.338892:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205103.338894:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205103.338897:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205103.338899:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205103.338902:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205103.338905:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929472, portal 4 00000100:00000001:0.0:1788205103.338906:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205103.338908:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205103.338913:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.338919:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205103.338921:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205103.338923:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205103.338925:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205929472, offset 0 00000400:00000200:0.0:1788205103.338929:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205103.338932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205103.338935:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205103.338938:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205103.338941:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205103.338943:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205103.338945:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205103.338948:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59117:59117:256:4294967295] 192.168.201.151@tcp LPNI seq info [59097:59097:8:4294967295] 00000400:00000200:0.0:1788205103.338953:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205103.338959:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205103.338962:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205103.338965:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000800:00000200:0.0:1788205103.338969:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205103.338972:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788205103.338976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205103.338979:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788205103.338982:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205103.338984:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:0.0:1788205103.338987:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205103.338990:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205103.338993:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:0.0:1788205103.338994:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:0.0:1788205103.338996:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205103.339000:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339007:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205103.339010:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205128 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339016:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.339018:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205103.339020:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205103.339023:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205103.339026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205103.339028:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301230640 00000100:00000001:0.0:1788205103.339030:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301234690 00000100:00000001:0.0:1788205103.339032:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301230640 00000100:00000001:0.0:1788205103.339034:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301234690 00000100:00000001:0.0:1788205103.339036:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205103.339038:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.339040:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205103.339043:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205103/real 1788205103] req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205103.339050:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205103.339051:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205103.339053:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205103.339055:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205103.339056:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205103.339058:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205103.339061:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205103.339069:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339075:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205103.339076:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339078:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205103.339079:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205103.339083:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339089:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205103.339093:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339099:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205103.339101:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339103:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205103.339105:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339107:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205103.339108:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205103.339724:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205103.339728:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205103.339786:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205103.339791:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205103.339800:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929472:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205103.339803:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339805:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205103.339808:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205929472/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205128 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205103.339815:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205103.339816:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205103.339818:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205103.339830:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205103.339831:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205103.339833:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205103.339835:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205103.339837:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205103.339981:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205103.339982:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205103.339984:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205103.339987:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205103.339988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339990:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205103.339994:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205103.339995:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205103.339997:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205103.339998:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529427:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205103.529434:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205103.529436:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205103.529439:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529444:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205103.529446:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529448:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529450:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529454:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205103.529456:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205103.529460:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205103.529462:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205103.529463:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529465:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205103.529466:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529468:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529469:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205103.529474:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205103.529475:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205103.529476:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529478:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205103.529479:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529481:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205103.529482:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205103.529793:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205103.529797:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205103.529799:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205103.848468:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205103.848478:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205103.848480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205103.848487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205103.848505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205103.848510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.848513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205103.848516:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205103.848519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.296827:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205104.296833:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205104.296834:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.296838:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205104.296840:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.296843:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.296845:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.296847:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.296849:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.360473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205104.360481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205104.360483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.360488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205104.360492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.360495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.360498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.360500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.360503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.363816:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205104.363822:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205104.363824:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205104.363827:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205104.363831:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205104.363834:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205104.363835:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205104.363837:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205104.363840:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205104.552498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205104.552504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205104.552508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205104.552508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205104.552510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205104.552511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205104.552513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205104.552514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205104.554779:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205104.554784:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205104.554787:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205104.873134:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205104.873142:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205104.873144:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.873149:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205104.873153:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.873156:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.873159:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.873161:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.873164:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205105.320454:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205105.320460:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205105.320463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205105.320469:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205105.320472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205105.320476:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.320479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205105.320481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.320484:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205105.384463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205105.384463:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205105.384470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205105.384471:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205105.384472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205105.384474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205105.384476:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:0.0:1788205105.384479:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205105.384480:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205105.384485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205105.384485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205105.384489:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.384489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205105.384492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.384495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205105.384496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205105.384499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205105.384502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205105.576461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205105.576466:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205105.576470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205105.576471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205105.576472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205105.576473:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205105.576476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205105.576477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205105.576751:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205105.576755:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205105.576758:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205105.896486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205105.896493:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205105.896495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205105.896500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205105.896503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205105.896507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.896509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205105.896517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205105.896520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205106.344471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205106.344478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205106.344480:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205106.344486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205106.344489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205106.344493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.344495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205106.344497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.344500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205106.408471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205106.408471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205106.408479:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205106.408479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205106.408484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205106.408485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205106.408491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205106.408492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205106.408497:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205106.408497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205106.408501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205106.408502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.408507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205106.408510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.408513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205106.408514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205106.408517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205106.408520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205106.600573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205106.600580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205106.600586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205106.600587:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205106.600590:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205106.600591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205106.600595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205106.600596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205106.600761:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205106.600764:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205106.600768:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205106.920729:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205106.920737:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205106.920739:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205106.920744:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205106.920748:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205106.920751:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.920753:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205106.920756:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205106.920759:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205107.368533:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205107.368541:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205107.368543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205107.368548:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205107.368557:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205107.368561:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205107.368564:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205107.368567:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205107.368570:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205107.432557:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205107.432564:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205107.432566:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205107.432571:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205107.432576:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205107.432580:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205107.432583:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205107.432586:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205107.432589:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205107.432844:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205107.432847:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205107.432848:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205107.432851:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205107.432853:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205107.432854:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205107.432856:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205107.432858:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205107.432860:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205107.624478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205107.624485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205107.624492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205107.624493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205107.624497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205107.624498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205107.624501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205107.624503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205107.624640:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205107.624643:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205107.624646:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205107.944512:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301234690, at 4301235200 00000100:00000010:0.0:1788205107.944526:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4720. 00000100:00000001:3.0:1788205107.944707:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205107.944711:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205107.944713:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205107.944718:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205107.944720:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205107.944724:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205107.944727:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205107.944729:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205107.944733:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301234690, at 4301235200 00000100:00000001:3.0:1788205107.944733:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205107.944742:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4420. 00000100:00000001:3.0:1788205108.392461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205108.392468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205108.392469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205108.392474:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205108.392477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205108.392481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.392484:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205108.392486:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.392488:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205108.456837:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205108.456847:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205108.456849:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205108.456853:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205108.456857:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205108.456864:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6563 00000400:00000200:2.0:1788205108.456872:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205108.456879:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205108.456882:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205108.456886:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.456889:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.456892:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.456895:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205108.456899:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205108.456901:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205108.456904:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788205108.456905:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000020:00000040:2.0:1788205108.456908:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:0.0:1788205108.456910:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1788205108.456912:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:0.0:1788205108.456914:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205108.456916:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00100000:0.0:1788205108.456919:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000040:2.0:1788205108.456922:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205108.456924:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205108.456926:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205108.456929:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205108.456930:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205108.456934:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205108.456934:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205108.456937:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00000001:0.0:1788205108.456938:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205108.456942:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205108.456943:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.456946:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.456950:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205108.456953:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205108.456956:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.456958:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205108.456961:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205108.456969:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205108.456972:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205108.456976:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205108.456978:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.456981:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205108.456984:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205108.456987:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205108.456996:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205108.457002:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205108.457010:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205108.457019:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205108.457023:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205108.457028:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.457049:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205108.457053:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205108.457056:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205108.457057:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205108.457059:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205108.457062:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205108.457064:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6563 00000400:00000200:2.0:1788205108.457068:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205108.457071:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205108.457073:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205108.457076:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457078:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.457080:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457081:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205108.457084:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205108.457085:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205108.457087:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205108.457090:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205108.457092:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205108.457094:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205108.457097:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205108.457098:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205108.457099:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205108.457103:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205108.457103:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205108.457107:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205108.457108:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205108.457112:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000040:0.0:1788205108.457117:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205108.457117:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457123:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.457128:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205108.457133:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788205108.457133:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205108.457135:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.457138:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.457141:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205108.457142:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1788205108.457145:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788205108.457148:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788205108.457149:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205108.457154:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205108.457156:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205929600:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205108.457158:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205108.457164:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205108.457165:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1788205108.457169:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205108.457171:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205108.457172:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205108.457175:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.457175:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.457181:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205108.457181:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205108.457183:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205108.457186:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205108.457188:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000010:2.0:1788205108.457191:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205108.457192:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457197:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205108.457201:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:2.0:1788205108.457206:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1788205108.457207:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205108.457215:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205108.457222:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929600, portal 25 00000100:00000001:0.0:1788205108.457227:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205108.457230:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788205108.457230:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205108.457234:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205108.457239:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.457248:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205108.457252:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205108.457253:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205108.457260:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205108.457264:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205108.457266:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205108.457269:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205108.457271:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1788205108.457273:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205108.457276:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217770. 00000100:00000200:0.0:1788205108.457279:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205929600, offset 0 00000100:00080000:2.0:1788205108.457283:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6563 00000400:00000200:0.0:1788205108.457289:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205108.457298:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.457303:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205108.457308:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205108.457312:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205108.457315:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205108.457317:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205108.457321:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59118:59118:256:4294967295] 192.168.201.151@tcp LPNI seq info [59098:59098:8:4294967295] 00000400:00000200:0.0:1788205108.457325:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.457334:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205108.457340:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:3.0:1788205108.457348:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000800:00000010:0.0:1788205108.457350:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af600. 00000100:00000001:3.0:1788205108.457352:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205108.457355:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205108.457359:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:3.0:1788205108.457359:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000800:00000200:0.1:1788205108.457382:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205108.457391:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205108.457395:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205108.457399:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301239809 00000100:00000010:0.0:1788205108.457403:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205108.457410:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205108.457410:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205108.457414:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.457418:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205108.457419:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301239810 00000100:00000001:3.0:1788205108.457420:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788205108.457425:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9720. 00000100:00000001:3.0:1788205108.457425:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457432:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205108.457436:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205108.457439:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457442:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457445:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205108.457452:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205108.457454:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205108.457456:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205108.457458:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205108.457459:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205108.457464:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.457471:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205108.457472:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.457474:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205108.457477:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205108.457479:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205108.457481:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929728:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205108.457484:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205108.457486:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205108.457487:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.457489:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205108.457492:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205108.457494:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205108.457497:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205108.457500:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da3c8. 00000400:00000010:0.0:1788205108.457503:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217440. 00000100:00000200:0.0:1788205108.457506:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929728, portal 10 00000100:00000001:0.0:1788205108.457508:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205108.457509:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205108.457516:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.457532:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205108.457535:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205108.457538:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217cc0. 00000100:00000200:0.0:1788205108.457540:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205929728, offset 0 00000400:00000200:0.0:1788205108.457545:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205108.457549:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.457553:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205108.457557:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205108.457560:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205108.457563:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205108.457565:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205108.457568:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59119:59119:255:4294967295] 192.168.201.151@tcp LPNI seq info [59099:59099:7:4294967295] 00000400:00000200:0.0:1788205108.457572:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.457577:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205108.457581:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205108.457584:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205108.457588:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205108.457591:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205108.457593:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205108.457597:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205108.457600:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205108.457602:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301235867 00000100:00000001:0.0:1788205108.457604:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301239809 00000100:00000001:0.0:1788205108.457606:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301235867 00000100:00000001:0.0:1788205108.457608:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301239810 00000100:00000001:0.0:1788205108.457610:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205108.457611:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457613:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457615:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205108.457618:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205108.457619:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205108.457620:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205108.457622:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205108.457623:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.457625:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205108.457672:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205108.457678:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205108.457680:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205108.457683:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457686:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.457688:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457690:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205108.457692:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205108.457694:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205108.457695:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205108.457697:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205108.457699:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205108.457701:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205108.457704:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457706:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.457708:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457709:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205108.457711:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205108.457714:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457716:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.457718:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205108.457721:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205108.457723:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457725:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205108.457728:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205108.457730:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457732:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205108.457735:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205108.457737:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205108.457739:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205108.457741:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457742:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205108.457744:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205108.457746:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205108.457749:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205108.457753:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205108.457812:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205108.457820:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205108.457822:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205108.457826:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.457842:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205108.457844:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205108.457848:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205108.457849:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205108.457851:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205108.457853:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205108.457856:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6563 00000400:00000200:2.0:1788205108.457860:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205108.457862:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205108.457864:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205108.457867:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457869:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.457871:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457873:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205108.457875:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205108.457876:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205108.457878:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205108.457880:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205108.457882:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205108.457884:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205108.457887:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457889:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205108.457890:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457892:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205108.457894:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205108.457896:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205108.457898:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205108.457900:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205108.457903:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205108.457905:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457906:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205108.457909:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205108.457911:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457912:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205108.457916:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205108.457918:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205108.457919:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205108.457921:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.457922:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205108.457924:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205108.457925:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205108.457929:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205108.457932:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205108.457935:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205108.457942:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205108.457944:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205108.457948:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.457954:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205108.457958:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6573) 00000100:00000001:0.0:1788205108.460704:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205108.460711:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205108.460713:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205108.460724:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.460735:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205108.460738:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.460741:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205108.460744:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205108.460746:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205108.460751:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205108.460754:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205108.460757:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205108.460759:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.460761:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205108.460767:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205108.460769:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205108.460774:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205108.460778:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da268. 00000400:00000010:0.0:1788205108.460784:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205108.460789:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929856, portal 4 00000100:00000001:0.0:1788205108.460791:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205108.460793:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788205108.460799:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.460805:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205108.460807:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205108.460810:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192175d8. 00000100:00000200:0.0:1788205108.460813:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205929856, offset 0 00000400:00000200:0.0:1788205108.460818:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205108.460825:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.460833:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205108.460838:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205108.460842:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205108.460845:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205108.460848:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205108.460852:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59120:59120:254:4294967295] 192.168.201.151@tcp LPNI seq info [59100:59100:6:4294967295] 00000400:00000200:0.0:1788205108.460856:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.460865:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205108.460870:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205108.460877:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788205108.460882:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205108.460887:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205108.460891:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205108.460894:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205108.460898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205108.460900:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301235756 00000100:00000001:0.0:1788205108.460903:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301239809 00000100:00000001:0.0:1788205108.460905:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301235756 00000100:00000001:0.0:1788205108.460908:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301239810 00000100:00000001:0.0:1788205108.460910:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205108.460912:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.460915:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205108.460919:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.460926:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205108.460927:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.460929:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205108.460931:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205108.460933:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205108.460936:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929984:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205108.460939:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205108.460941:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205108.460942:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205108.460944:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205108.460948:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205108.460950:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205108.460952:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205108.460955:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205108.460958:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205108.460961:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205929984, portal 4 00000100:00000001:0.0:1788205108.460963:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205108.460965:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788205108.460984:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205108.460997:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205108.460999:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205108.461002:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217000. 00000100:00000200:0.0:1788205108.461005:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205929984, offset 0 00000400:00000200:0.0:1788205108.461008:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205108.461012:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.461015:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205108.461018:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205108.461020:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205108.461022:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205108.461024:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205108.461027:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59121:59121:253:4294967295] 192.168.201.151@tcp LPNI seq info [59101:59101:5:4294967295] 00000400:00000200:0.0:1788205108.461030:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205108.461036:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205108.461046:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205108.461049:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054afb00. 00000800:00000200:0.0:1788205108.461053:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205108.461056:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205108.461058:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205108.461061:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205108.461063:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205108.461065:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301235762 00000100:00000001:0.0:1788205108.461067:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301239809 00000100:00000001:0.0:1788205108.461070:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301235762 00000100:00000001:0.0:1788205108.461071:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301239810 00000100:00000001:0.0:1788205108.461073:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205108.461075:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.461078:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.461080:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205108.461087:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205108.461089:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205108.461091:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205108.461092:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205108.461093:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205108.461095:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205108.648525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205108.648532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205108.648537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205108.648538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205108.648542:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205108.648543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205108.648546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205108.648547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205108.648603:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205108.648606:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205108.648610:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205108.968956:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205108.968961:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205108.968963:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205108.968967:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205108.968971:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205108.968974:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205108.968977:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205108.968979:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205108.968981:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205109.416486:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205109.416494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205109.416496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205109.416502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205109.416504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205109.416507:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.416515:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205109.416518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.416521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205109.480485:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205109.480492:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205109.480495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205109.480499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205109.480502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205109.480506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.480521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205109.480524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.480527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205109.483626:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205109.483649:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205109.483652:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205109.483657:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205109.483661:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205109.483664:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205109.483667:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205109.483669:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205109.483672:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205109.672498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205109.672506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205109.672512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205109.672513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205109.672516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205109.672517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205109.672520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205109.672522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205109.672715:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205109.672719:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205109.672722:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205109.992577:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205109.992584:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205109.992587:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205109.992593:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205109.992596:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205109.992600:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205109.992603:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205109.992605:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205109.992608:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205110.443564:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205110.443571:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205110.443573:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205110.443579:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205110.443582:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205110.443586:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.443588:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205110.443591:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.443594:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205110.504502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205110.504509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205110.504512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205110.504517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205110.504521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205110.504524:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.504527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205110.504531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.504534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.504651:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205110.504655:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205110.504656:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.504660:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205110.504662:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205110.504664:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205110.504666:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205110.504668:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205110.504670:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205110.696543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205110.696547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205110.696551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205110.696551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205110.696554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205110.696555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205110.696556:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205110.696557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205110.696791:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205110.696795:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205110.696798:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205111.016505:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205111.016513:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205111.016514:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205111.016520:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205111.016523:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.016527:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.016530:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205111.016533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.016536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205111.464460:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205111.464466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205111.464467:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205111.464472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205111.464474:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205111.464477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.464479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205111.464481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.464482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205111.528479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205111.528486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205111.528489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205111.528494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205111.528498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205111.528501:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.528504:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205111.528507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.528509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.528660:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205111.528662:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205111.528663:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205111.528665:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205111.528667:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.528668:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.528669:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205111.528671:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.528672:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1788205111.529217:0:2333:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:1.0:1788205111.529255:0:2333:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1788205111.529265:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205111.529269:0:2333:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:1.0:1788205111.529274:0:2333:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205111.529278:0:2333:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1788205111.529283:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788205111.529287:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af600. 00000400:00000200:1.0:1788205111.529293:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788205111.529297:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205111.529304:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217770 00000400:00000010:1.0:1788205111.529307:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217770. 00000100:00000001:1.0:1788205111.529311:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205111.529320:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.529346:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205111.529350:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.529358:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205111.529360:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788205111.529402:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788205111.529405:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:1.0:1788205111.529408:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788205111.529412:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205111.529415:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217cc0 00000400:00000010:1.0:1788205111.529417:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217cc0. 00000100:00000001:1.0:1788205111.529419:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205111.529424:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.529432:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205111.529434:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.529441:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205111.529442:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788205111.529447:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788205111.529450:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:1.0:1788205111.529452:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788205111.529456:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205111.529459:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192175d8 00000400:00000010:1.0:1788205111.529461:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192175d8. 00000100:00000001:1.0:1788205111.529463:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205111.529469:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529475:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205111.529480:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.529485:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788205111.529489:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1788205111.529493:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205108/real 1788205111] req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205111.529497:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205111.529510:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205111.529514:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788205111.529514:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205111.529517:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da3c8. 00000100:00000001:1.0:1788205111.529518:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1788205111.529522:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217440 00000400:00000010:0.0:1788205111.529526:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217440. 00000800:00000100:1.0:1788205111.529528:0:2333:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000100:00000001:0.0:1788205111.529530:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205111.529536:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205111.529541:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1788205111.529545:0:2333:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054afb00. 00000100:00000001:0.0:1788205111.529546:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205111.529548:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529551:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529552:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205111.529553:0:2333:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:0.0:1788205111.529556:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205111.529563:0:2333:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1788205111.529567:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205111.529569:0:2333:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217000 00000400:00000010:1.0:1788205111.529572:0:2333:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217000. 00000100:00000040:0.0:1788205111.529575:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.529577:0:2333:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205111.529584:0:2333:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529588:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205111.529591:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529593:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529594:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529597:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205111.529626:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205111.529629:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205111.529633:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205111.529636:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205111.529639:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205111.529645:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929728:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205111.529649:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205111.529653:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205108/real 1788205111] req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205111.529659:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205111.529660:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205111.529662:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:0.0:1788205111.529664:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:0.0:1788205111.529665:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:0.0:1788205111.529667:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205111.529669:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205111.529673:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529677:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205111.529677:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529678:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529679:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205111.529681:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529685:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205111.529687:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529691:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205111.529692:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529693:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529694:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529695:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205111.529696:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205111.529697:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205111.529699:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205111.529700:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205111.529703:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205111.529709:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205929600:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205111.529712:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.529714:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205111.529716:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205929728/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529723:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205111.529725:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205111.529728:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205111.529732:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205111.529735:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205111.529736:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205111.529739:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205111.529741:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205111.529745:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205111.529746:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205111.529748:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205111.529751:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205111.529752:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529753:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205111.529755:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205929600/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529762:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205111.529763:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205111.529764:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205111.529766:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205111.529766:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205111.529767:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205111.529768:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205111.529769:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205111.529771:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205111.529771:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205111.529772:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205111.529773:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205111.529774:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529775:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.529777:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205111.529777:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.529778:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205111.529781:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205108/real 1788205111] req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205111.529786:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205111.529787:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205111.529788:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da268. 00000400:00000200:0.0:1788205111.529789:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205111.529790:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205111.529792:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205111.529793:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205111.529798:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529802:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205111.529802:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529803:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529804:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205111.529806:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529810:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205111.529812:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529816:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205111.529817:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529818:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.529819:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529820:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205111.529821:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205111.529822:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205111.529824:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205111.529825:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205111.529827:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205111.529831:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205111.529834:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.529835:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205111.529836:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205929856/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.529840:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205111.529840:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205111.529841:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205111.529843:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205111.529843:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205111.529844:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205111.529845:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205111.529846:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205111.529847:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205111.529848:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205111.529849:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205111.529850:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205111.529851:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.529852:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.529853:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205111.529854:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.529854:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.529855:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205111.533139:0:2333:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205111.533150:0:2333:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205133 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205111.533159:0:2333:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205111.533161:0:2333:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205111.533655:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205111.533659:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205111.533661:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205111.533669:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205108/real 1788205111] req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205111.533682:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205111.533684:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205111.533687:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205111.533692:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205111.533694:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205111.533698:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205111.533702:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205111.533710:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.533716:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205111.533718:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533720:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.533722:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205111.533726:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.533732:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205111.533737:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.533745:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205111.533748:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533750:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205111.533751:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533753:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205111.533755:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205111.533759:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205111.533764:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205111.533766:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205111.533770:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205111.533777:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205929984:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205111.533781:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533783:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205111.533786:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205929984/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205133 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205111.533792:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205111.533794:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205111.533798:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205111.533803:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205111.533806:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205111.533808:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205111.533810:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205111.533812:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:0.0:1788205111.533816:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205111.533817:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205111.533819:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205111.533823:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205111.533824:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533827:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205111.533834:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205111.533835:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205111.533838:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205111.533839:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721041:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205111.721048:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721053:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205111.721056:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205111.721059:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205111.721062:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721065:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205111.721067:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721069:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721071:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721075:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205111.721076:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205111.721079:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205111.721081:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205111.721147:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205111.721151:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205111.721154:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205112.040527:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205112.040535:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205112.040538:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205112.040543:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205112.040546:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205112.040550:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205112.040641:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205112.040645:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205112.040649:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205112.488476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205112.488482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205112.488484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205112.488489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205112.488492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205112.488496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.488498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205112.488500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.488503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205112.552691:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205112.552698:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205112.552700:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205112.552704:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205112.552705:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205112.552707:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205112.552711:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205112.552712:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205112.552718:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205112.552718:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205112.552724:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.552725:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.552731:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205112.552733:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.552736:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205112.552739:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205112.552743:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205112.552746:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205112.744498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205112.744504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205112.744509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205112.744510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205112.744513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205112.744514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205112.744516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205112.744517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205112.744646:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205112.744650:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205112.744653:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205113.064526:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301239810, at 4301240320 00000100:00000010:1.0:1788205113.064542:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9720. 00000100:00000040:0.0:1788205113.065788:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301239809, at 4301240321 00000100:00000010:0.0:1788205113.065798:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205113.066053:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205113.066057:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205113.066059:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205113.066063:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205113.066065:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205113.066068:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.066071:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205113.066074:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.066077:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205113.512879:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205113.512884:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205113.512885:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205113.512889:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205113.512892:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205113.512895:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.512896:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205113.512898:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.512900:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.576488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205113.576495:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205113.576497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205113.576503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205113.576507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205113.576511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.576514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205113.576516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.576519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205113.576672:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205113.576674:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205113.576676:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205113.576678:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205113.576679:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205113.576681:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.576682:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205113.576683:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205113.576684:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205113.582981:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205113.582990:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205113.582993:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205113.582997:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205113.583000:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205113.583007:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6568 00000400:00000200:1.0:1788205113.583016:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205113.583022:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205113.583025:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205113.583028:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583032:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583034:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583036:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205113.583040:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205113.583042:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205113.583044:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205113.583047:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205113.583049:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205113.583051:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205113.583054:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583056:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583058:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583060:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205113.583063:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205113.583066:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583068:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205113.583070:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205113.583073:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205113.583075:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583078:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205113.583080:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205113.583089:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788205113.583092:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205113.583096:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205113.583098:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583100:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205113.583103:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205113.583107:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205113.583114:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788205113.583132:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205113.583142:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205113.583152:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205113.583156:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205113.583160:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205113.583180:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205113.583184:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205113.583197:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205113.583199:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205113.583201:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205113.583204:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205113.583207:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6568 00000400:00000200:1.0:1788205113.583212:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205113.583215:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205113.583217:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205113.583220:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583222:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583225:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583227:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205113.583229:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205113.583231:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205113.583232:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205113.583234:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:0.0:1788205113.583237:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788205113.583237:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205113.583242:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205113.583243:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205113.583246:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205113.583248:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205113.583252:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583257:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205113.583259:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205113.583259:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205113.583264:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205113.583270:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583276:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.583278:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788205113.583279:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205113.583282:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205113.583286:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205113.583287:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205113.583292:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205113.583293:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.583298:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583299:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205113.583305:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205113.583306:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205930112:192.168.201.151@tcp:250:kworker.0 00000008:00000001:1.0:1788205113.583311:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.583314:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205113.583316:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.583319:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1788205113.583322:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 02000000:00000001:0.0:1788205113.583324:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205113.583326:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205113.583328:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205113.583332:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205113.583334:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205113.583336:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 00000100:00000001:1.0:1788205113.583337:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205113.583340:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205113.583341:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205113.583343:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.583349:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:1.0:1788205113.583350:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000400:00000010:0.0:1788205113.583355:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000100:00000001:1.0:1788205113.583356:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205113.583382:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205113.583390:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1788205113.583392:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000001:1.0:1788205113.583393:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205113.583400:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205113.583402:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930112, portal 25 00000100:00000001:0.0:1788205113.583409:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205113.583413:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00000001:1.0:1788205113.583415:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205113.583420:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1788205113.583423:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205113.583424:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205113.583428:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205113.583430:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205113.583437:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205113.583440:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788205113.583444:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6568 00000100:00000040:0.0:1788205113.583448:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205113.583453:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000400:00000200:1.0:1788205113.583453:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:0.0:1788205113.583460:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205930112, offset 0 00000400:00000200:1.0:1788205113.583460:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205113.583465:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205113.583470:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205113.583473:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583474:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583479:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583481:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205113.583485:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788205113.583485:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205113.583487:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205113.583491:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205113.583493:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:0.0:1788205113.583494:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:1.0:1788205113.583497:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205113.583501:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205113.583505:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583506:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.583508:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583510:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205113.583513:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:0.0:1788205113.583515:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1788205113.583517:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205113.583522:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.583525:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:1.0:1788205113.583525:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205113.583531:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205113.583532:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1788205113.583535:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583539:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205113.583541:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205113.583543:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583544:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205113.583548:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788205113.583550:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205113.583553:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205113.583554:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.583556:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205113.583557:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205113.583559:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205113.583575:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788205113.583579:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205113.583584:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205113.583592:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205113.583594:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205113.583597:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205113.583605:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205113.583607:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205113.583611:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205113.583612:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205113.583614:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205113.583616:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205113.583619:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6568 00000400:00000200:0.0:1788205113.583652:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205113.583657:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59122:59122:256:4294967295] 192.168.201.151@tcp LPNI seq info [59102:59102:8:4294967295] 00000400:00000200:0.0:1788205113.583662:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.583672:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205113.583678:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.583683:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205113.583690:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205113.583693:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205113.583699:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205113.583715:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205113.583719:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.583722:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205113.583730:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205113.583734:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205113.584577:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205113.584582:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205113.584585:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205113.584588:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.584590:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.584592:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.584594:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205113.584596:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205113.584598:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205113.584599:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205113.584601:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205113.584603:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205113.584605:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205113.584608:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.584610:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205113.584611:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205113.584613:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205113.584615:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205113.584617:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205113.584619:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205113.584621:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205113.584635:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205113.584637:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.584639:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205113.584641:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205113.584644:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.584645:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205113.584649:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788205113.584652:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205113.584654:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205113.584655:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205113.584657:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205113.584658:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205113.584660:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205113.584664:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788205113.584668:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205113.584673:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205113.584708:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205113.584712:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205113.584716:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205113.584721:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584731:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.584735:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584742:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.584744:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205113.584751:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.584755:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205113.584758:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301244937 00000100:00000010:0.0:1788205113.584763:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205113.584769:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205113.584776:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301244937 00000100:00000010:0.0:1788205113.584780:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff92a0. 00000100:00000001:0.0:1788205113.584783:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205113.584786:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205113.584788:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.584791:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205113.584795:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205113/real 1788205113] req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205113.584804:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205113.584806:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205113.584808:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205113.584811:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205113.584813:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205113.584815:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205113.584819:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205113.584826:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584833:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205113.584834:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584836:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.584838:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205113.584842:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584848:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205113.584852:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584859:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205113.584862:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584863:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.584864:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584867:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205113.584869:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205113.584872:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205113.584876:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205113.584878:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205113.584882:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205113.584891:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205930112:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205113.584895:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584897:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.584901:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205930112/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584907:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205113.584909:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205113.584911:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205113.584915:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205113.584917:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205113.584918:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205113.584920:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205113.584922:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:0.0:1788205113.584926:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205113.584927:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205113.584929:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:0.0:1788205113.584933:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205113.584934:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584936:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.584943:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205113.584945:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.584947:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205113.584948:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205113.584951:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205113.584956:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1788205113.584957:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205113.584961:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205113.584970:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.584980:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205113.584982:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.584998:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205113.585001:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205113.585003:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205113.585007:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930240:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205113.585010:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205113.585013:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205113.585014:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.585016:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585019:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205113.585021:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.585025:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205113.585027:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205113.585031:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205113.585035:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930240, portal 10 00000100:00000001:0.0:1788205113.585040:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205113.585042:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:0.0:1788205113.585052:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205113.585052:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205113.585061:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6578) 00000100:00000001:0.0:1788205113.585062:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205113.585068:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205113.585071:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205113.585074:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205930240, offset 0 00000400:00000200:0.0:1788205113.585079:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585084:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585089:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205113.585094:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205113.585097:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205113.585100:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205113.585102:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585106:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59123:59123:256:4294967295] 192.168.201.151@tcp LPNI seq info [59103:59103:8:4294967295] 00000400:00000200:0.0:1788205113.585110:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585117:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205113.585122:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585126:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205113.585130:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205113.585132:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205113.585137:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205113.585140:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205113.585144:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585146:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205113.585150:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205113.585154:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205113.585157:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205113.585158:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205113.585161:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205113.585164:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585172:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585175:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585181:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585183:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205113.585186:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205113.585189:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205113.585192:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205113.585194:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301240994 00000100:00000001:0.0:1788205113.585196:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301240994 00000100:00000001:0.0:1788205113.585200:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585202:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205113.585204:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585206:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205113.585210:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205113/real 1788205113] req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205113.585217:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205113.585219:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205113.585220:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205113.585223:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205113.585225:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205113.585227:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205113.585230:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205113.585238:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585245:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205113.585247:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585248:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585250:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205113.585253:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585259:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205113.585263:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585270:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205113.585272:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585273:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585275:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585276:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205113.585279:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205113.585281:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205113.585284:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205113.585286:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205113.585291:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205113.585299:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930240:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205113.585302:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205113.585305:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585312:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205113.585314:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.585316:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205113.585318:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205113.585320:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205113.585322:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205113.585325:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205113.585327:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205113.585329:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.585330:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585334:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205113.585336:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.585339:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205113.585342:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205113.585345:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205113.585347:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930368, portal 4 00000100:00000001:0.0:1788205113.585349:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205113.585350:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:0.0:1788205113.585356:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585362:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205113.585383:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205113.585386:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205113.585389:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205930368, offset 0 00000400:00000200:0.0:1788205113.585393:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585396:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585400:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205113.585402:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205113.585405:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205113.585407:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205113.585409:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585412:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59124:59124:256:4294967295] 192.168.201.151@tcp LPNI seq info [59104:59104:8:4294967295] 00000400:00000200:0.0:1788205113.585416:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585422:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205113.585427:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585431:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205113.585434:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205113.585437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205113.585441:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205113.585444:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205113.585447:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585449:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205113.585452:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205113.585455:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205113.585458:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205113.585459:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205113.585462:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205113.585466:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585472:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585474:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585480:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585482:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205113.585485:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205113.585487:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205113.585490:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205113.585493:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301240881 00000100:00000001:0.0:1788205113.585495:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585497:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301240881 00000100:00000001:0.0:1788205113.585498:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585500:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205113.585502:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585504:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205113.585507:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205113/real 1788205113] req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205113.585515:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205113.585516:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205113.585517:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205113.585520:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205113.585521:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205113.585523:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205113.585525:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205113.585532:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585537:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205113.585538:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585539:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585540:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205113.585543:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585549:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205113.585552:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585558:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205113.585560:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585573:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585575:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585576:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205113.585578:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205113.585581:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205113.585584:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205113.585586:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205113.585590:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205113.585597:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205113.585601:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585603:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585606:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205930240/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585612:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205113.585613:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585615:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205113.585618:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205113.585620:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205113.585622:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205113.585624:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205113.585626:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:0.0:1788205113.585628:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205113.585629:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205113.585631:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:0.0:1788205113.585634:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205113.585635:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585637:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585640:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205930368/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585647:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205113.585648:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585649:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205113.585651:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205113.585652:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205113.585653:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205113.585655:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205113.585656:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:0.0:1788205113.585658:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205113.585659:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205113.585660:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:0.0:1788205113.585663:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205113.585664:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585665:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585669:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205113.585671:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585673:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205113.585674:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205113.585676:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205113.585679:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585685:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205113.585687:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.585689:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205113.585690:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205113.585697:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205113.585700:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930496:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205113.585703:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205113.585705:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205113.585706:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205113.585708:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585711:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205113.585713:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205113.585716:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205113.585718:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205113.585722:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205113.585724:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930496, portal 4 00000100:00000001:0.0:1788205113.585726:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205113.585727:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:0.0:1788205113.585733:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585739:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205113.585741:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205113.585744:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205113.585746:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205930496, offset 0 00000400:00000200:0.0:1788205113.585749:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585753:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585756:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205113.585759:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205113.585762:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205113.585764:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205113.585766:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205113.585769:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59125:59125:256:4294967295] 192.168.201.151@tcp LPNI seq info [59105:59105:8:4294967295] 00000400:00000200:0.0:1788205113.585773:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205113.585778:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205113.585782:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585785:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205113.585788:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205113.585791:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205113.585795:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205113.585798:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205113.585801:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205113.585802:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205113.585806:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205113.585809:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205113.585812:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205113.585814:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205113.585816:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205113.585819:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585826:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585829:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585835:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585837:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205113.585840:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205113.585842:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205113.585845:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205113.585847:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301240887 00000100:00000001:0.0:1788205113.585849:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585850:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301240887 00000100:00000001:0.0:1788205113.585852:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301244937 00000100:00000001:0.0:1788205113.585854:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205113.585856:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585858:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205113.585862:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205113/real 1788205113] req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205113.585868:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205113.585869:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205113.585871:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205113.585873:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205113.585875:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205113.585876:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205113.585879:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205113.585892:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585898:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205113.585900:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585901:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585903:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205113.585906:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585912:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205113.585916:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585922:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205113.585924:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585926:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205113.585927:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585929:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205113.585931:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205113.585933:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205113.585936:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205113.585939:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205113.585942:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205113.585949:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930496:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205113.585952:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585954:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205113.585957:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205930496/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205113.585968:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205113.585970:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205113.585971:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205113.585974:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205113.585976:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205113.585977:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205113.585979:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205113.585980:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:0.0:1788205113.585982:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205113.585983:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205113.585985:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:0.0:1788205113.585987:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205113.585988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585990:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205113.585993:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205113.585994:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205113.585996:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205113.585997:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205113.768510:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205113.768513:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205113.768517:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768522:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205113.768524:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768526:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768528:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205113.768534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205113.768539:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205113.768540:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205113.768542:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768544:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205113.768546:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768547:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.768549:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.769088:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205113.769092:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205113.769093:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205113.769095:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.769097:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205113.769098:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.769100:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205113.769101:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205113.769166:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205113.769170:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205113.769172:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205114.088682:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205114.088689:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205114.088691:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205114.088696:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205114.088700:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205114.088704:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.088706:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205114.088709:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.088712:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205114.536475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205114.536481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205114.536482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205114.536486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205114.536488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205114.536490:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.536492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205114.536494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.536496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205114.600472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205114.600480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205114.600484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205114.600484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205114.600488:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1788205114.600490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205114.600490:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205114.600495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205114.600496:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205114.600498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205114.600501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.600502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205114.600508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205114.600511:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205114.600515:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205114.600515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205114.600518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.600521:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205114.792511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205114.792526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205114.792531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205114.792532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205114.792534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205114.792534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205114.792537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205114.792538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205114.792682:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205114.792685:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205114.792687:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205115.112430:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205115.112437:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205115.112439:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.112444:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205115.112448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205115.112451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.112454:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205115.112457:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.112459:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.560483:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205115.560491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205115.560493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.560498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205115.560501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205115.560506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.560508:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205115.560510:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.560513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205115.624462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205115.624462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205115.624470:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205115.624470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205115.624473:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.624473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205115.624478:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205115.624482:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205115.624483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205115.624485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.624486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205115.624490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205115.624490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205115.624494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205115.624498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.624498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205115.624501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.624503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205115.816559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205115.816566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205115.816572:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205115.816573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205115.816576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205115.816577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205115.816581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205115.816582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205115.816645:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205115.816649:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205115.816653:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205116.137886:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205116.137893:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205116.137895:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205116.137901:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205116.137904:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205116.137908:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205116.137911:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205116.137914:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205116.137917:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205116.584470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205116.584476:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205116.584478:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205116.584483:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205116.584485:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205116.584488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.584490:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205116.584492:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.584493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205116.648484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205116.648493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205116.648495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205116.648500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205116.648503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205116.648508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205116.648522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205116.648525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205116.648528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205116.648553:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205116.648558:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205116.648559:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205116.648562:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205116.648564:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205116.648566:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.648568:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205116.648570:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.648572:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205116.840551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205116.840557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205116.840562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205116.840563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205116.840567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205116.840568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205116.840572:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205116.840573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205116.840627:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205116.840629:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205116.840631:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205117.160473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205117.160480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205117.160482:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205117.160487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205117.160491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205117.160494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205117.160497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205117.160499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205117.160518:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.608474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205117.608481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205117.608482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205117.608488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205117.608491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205117.608506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.608509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205117.608512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.608514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.672503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205117.672505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205117.672512:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205117.672513:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205117.672515:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.672517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205117.672519:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205117.672521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205117.672522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205117.672527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205117.672528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.672531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205117.672532:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205117.672534:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205117.672537:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.672538:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205117.672541:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.672544:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205117.864464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205117.864471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205117.864476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205117.864478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205117.864481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205117.864482:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205117.864485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205117.864486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205117.864565:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205117.864569:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205117.864571:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205118.184466:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301244937, at 4301245440 00000100:00000010:1.0:1788205118.184480:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff92a0. 00000100:00000001:3.0:1788205118.184492:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205118.184497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205118.184500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.184504:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205118.184507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205118.184511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205118.184514:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205118.184517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205118.184519:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301244937, at 4301245440 00000100:00000001:3.0:1788205118.184521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205118.184529:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205118.632725:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205118.632733:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205118.632735:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.632741:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205118.632744:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205118.632748:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205118.632750:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205118.632752:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205118.632756:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.696564:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205118.696571:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205118.696574:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.696579:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205118.696583:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205118.696587:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.696590:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.696593:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.696596:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.696623:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205118.696625:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205118.696626:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.696628:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205118.696630:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205118.696632:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.696634:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.696635:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.696637:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205118.696671:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205118.696677:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205118.696680:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205118.696683:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205118.696688:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205118.696696:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6573 00000400:00000200:3.0:1788205118.696706:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205118.696712:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205118.696716:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205118.696720:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.696724:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.696727:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.696729:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205118.696733:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205118.696735:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205118.696738:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205118.696741:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205118.696743:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205118.696746:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205118.696749:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.696751:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.696753:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.696755:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205118.696758:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205118.696761:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205118.696763:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205118.696766:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205118.696769:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205118.696773:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.696775:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205118.696777:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205118.696786:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0000. 00000020:00000040:3.0:1788205118.696790:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205118.696794:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205118.696796:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.696799:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205118.696801:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205118.696804:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205118.696812:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20c00. 00000100:00000001:3.0:1788205118.696981:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.696993:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205118.697010:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205118.697014:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205118.697019:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee757f9f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697080:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205118.697083:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205118.697085:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205118.697090:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697098:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.697111:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.697114:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205118.697117:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205118.697119:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.697123:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205930624:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205118.697126:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205118.697130:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205118.697132:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.697134:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205118.697139:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.697141:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205118.697145:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205118.697149:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205118.697155:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205118.697162:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930624, portal 25 00000100:00000001:0.0:1788205118.697164:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205118.697166:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:0.0:1788205118.697171:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697178:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205118.697181:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205118.697184:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205118.697187:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205930624, offset 0 00000400:00000200:0.0:1788205118.697192:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205118.697200:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.697206:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205118.697211:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205118.697215:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205118.697219:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205118.697221:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205118.697225:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59126:59126:256:4294967295] 192.168.201.151@tcp LPNI seq info [59106:59106:8:4294967295] 00000400:00000200:0.0:1788205118.697229:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.697238:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205118.697244:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.697249:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205118.697255:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205118.697258:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205118.697263:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205118.697266:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205118.697270:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.697273:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205118.697281:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205118.697286:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205118.697291:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205118.697294:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205118.697298:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205118.697302:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205118.697396:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205118.697400:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205118.697403:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205118.697405:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205118.697406:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205118.697409:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205118.697412:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6573 00000400:00000200:3.0:1788205118.697416:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205118.697419:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205118.697421:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205118.697423:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.697425:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.697427:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.697429:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205118.697431:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205118.697432:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205118.697434:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205118.697436:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205118.697438:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205118.697440:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205118.697443:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.697445:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.697447:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.697449:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205118.697451:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788205118.697454:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205118.697456:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205118.697458:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205118.697463:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205118.697465:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.697482:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205118.697486:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205118.697489:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.697491:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205118.697495:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3480. 00000020:00000040:3.0:1788205118.697497:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205118.697500:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205118.697501:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.697503:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205118.697504:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205118.697506:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205118.697510:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22400. 00000100:00000001:3.0:1788205118.697513:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.697518:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205118.697526:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205118.697528:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205118.697532:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133b1fcc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697701:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.697707:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697714:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.697716:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205118.697722:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205118.697726:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205118.697729:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301250050 00000100:00000010:0.0:1788205118.697734:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205118.697740:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205118.697745:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301250050 00000100:00000010:0.0:1788205118.697748:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff97e0. 00000100:00000001:0.0:1788205118.697751:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205118.697754:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205118.697757:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.697759:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205118.697763:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205118/real 1788205118] req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205118.697771:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205118.697773:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205118.697775:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205118.697778:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205118.697779:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205118.697781:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205118.697785:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205118.697794:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697800:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205118.697802:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697804:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.697806:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205118.697809:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697816:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205118.697825:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697832:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205118.697834:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697836:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.697837:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697839:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205118.697842:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205118.697845:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205118.697849:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205118.697852:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205118.697856:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205118.697864:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205930624:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205118.697869:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697871:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.697874:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205930624/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697885:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205118.697888:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205118.697890:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.697893:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205118.697895:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205118.697896:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205118.697898:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205118.697901:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20c00. 02000000:00000001:0.0:1788205118.697904:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205118.697905:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205118.697907:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:0.0:1788205118.697915:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205118.697917:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697919:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.697925:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205118.697927:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.697929:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205118.697930:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205118.697931:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205118.697935:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697940:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.697942:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.697944:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205118.697945:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205118.697947:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.697950:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930752:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205118.697953:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205118.697955:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205118.697956:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.697958:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205118.697961:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.697963:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205118.697966:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205118.697968:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205118.697971:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205118.697974:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930752, portal 10 00000100:00000001:0.0:1788205118.697975:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205118.697976:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:0.0:1788205118.697981:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.697987:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205118.697989:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205118.697990:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205118.697992:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205930752, offset 0 00000400:00000200:0.0:1788205118.697996:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205118.697999:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.698003:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205118.698006:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205118.698008:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205118.698010:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205118.698012:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205118.698016:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59127:59127:256:4294967295] 192.168.201.151@tcp LPNI seq info [59107:59107:8:4294967295] 00000400:00000200:0.0:1788205118.698019:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.698026:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205118.698030:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.698033:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205118.698036:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205118.698039:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205118.698042:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205118.698045:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205118.698049:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.698051:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205118.698054:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205118.698058:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205118.698061:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205118.698062:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205118.698065:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205118.698068:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698075:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.698077:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698084:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.698085:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205118.698088:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205118.698091:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205118.698095:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205118.698097:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301246107 00000100:00000001:0.0:1788205118.698112:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.698114:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301246107 00000100:00000001:0.0:1788205118.698116:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.698117:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205118.698119:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.698121:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205118.698124:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205118/real 1788205118] req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205118.698132:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205118.698133:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205118.698135:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205118.698137:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205118.698139:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205118.698141:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205118.698144:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205118.698152:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698164:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205118.698166:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698167:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.698169:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205118.698172:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698179:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205118.698182:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698188:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205118.698191:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698192:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.698193:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698195:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205118.698197:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205118.698199:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205118.698202:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205118.698204:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205118.698207:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205118.698214:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930752:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205118.698217:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698219:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.698222:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205930752/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698228:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205118.698230:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205118.698231:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.698234:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205118.698236:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205118.698237:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205118.698239:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205118.698241:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22400. 02000000:00000001:0.0:1788205118.698243:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205118.698244:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205118.698245:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:0.0:1788205118.698248:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205118.698249:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698251:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.698254:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205118.698255:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.698257:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205118.698259:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698558:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205118.698561:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205118.698564:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205118.698566:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205118.698567:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205118.698570:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205118.698573:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6573 00000400:00000200:3.0:1788205118.698577:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205118.698580:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205118.698582:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205118.698585:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.698590:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698592:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205118.698595:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205118.698596:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205118.698597:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205118.698599:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788205118.698602:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205118.698604:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788205118.698606:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698608:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.698610:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698612:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205118.698613:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205118.698616:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698618:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205118.698620:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205118.698623:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205118.698625:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698627:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205118.698630:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205118.698632:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698634:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205118.698638:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2d80. 00000020:00000040:3.0:1788205118.698640:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205118.698644:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205118.698645:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698647:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205118.698648:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205118.698650:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205118.698654:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23c00. 00000100:00000001:3.0:1788205118.698658:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.698663:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205118.698669:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205118.698672:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205118.698675:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001291af88] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205118.698683:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205118.698686:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205118.698689:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205118.698690:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205118.698692:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205118.698694:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205118.698696:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6573 00000400:00000200:3.0:1788205118.698700:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205118.698703:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205118.698705:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205118.698707:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698709:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.698710:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698712:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205118.698715:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205118.698716:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205118.698717:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205118.698719:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205118.698721:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205118.698723:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205118.698726:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698727:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205118.698729:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698731:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205118.698733:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205118.698735:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205118.698737:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205118.698739:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205118.698741:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205118.698743:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698745:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205118.698747:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788205118.698750:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698751:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205118.698754:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3100. 00000020:00000040:3.0:1788205118.698756:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205118.698759:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205118.698760:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205118.698761:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205118.698762:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205118.698764:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205118.698767:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22000. 00000100:00000001:3.0:1788205118.698771:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.698775:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205118.698781:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205118.698783:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205118.698786:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e74a292a] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205118.698793:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205118.698798:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6583) 00000100:00000001:0.0:1788205118.698970:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205118.698973:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205118.698974:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205118.698979:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.698986:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.698987:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.698990:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205118.698992:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205118.698994:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.698997:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205118.699000:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205118.699002:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205118.699004:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.699005:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205118.699009:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.699011:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205118.699015:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205118.699018:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205118.699022:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205118.699025:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205930880, portal 4 00000100:00000001:0.0:1788205118.699028:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205118.699029:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:0.0:1788205118.699034:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699040:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205118.699043:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205118.699046:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205118.699048:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205930880, offset 0 00000400:00000200:0.0:1788205118.699052:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205118.699056:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.699060:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205118.699063:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205118.699066:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205118.699068:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205118.699071:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205118.699074:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59128:59128:256:4294967295] 192.168.201.151@tcp LPNI seq info [59108:59108:8:4294967295] 00000400:00000200:0.0:1788205118.699078:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.699084:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205118.699089:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.699092:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205118.699096:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205118.699191:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205118.699198:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205118.699201:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205118.699204:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.699206:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205118.699210:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205118.699214:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205118.699217:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205118.699219:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205118.699222:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205118.699226:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699234:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.699237:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699243:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.699244:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205118.699248:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205118.699251:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205118.699255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205118.699257:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301245994 00000100:00000001:0.0:1788205118.699259:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.699261:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301245994 00000100:00000001:0.0:1788205118.699263:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.699265:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205118.699267:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.699269:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205118.699274:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205118/real 1788205118] req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205118.699281:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205118.699283:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205118.699284:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205118.699287:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205118.699288:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205118.699290:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205118.699294:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205118.699301:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699307:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205118.699309:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699310:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.699312:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205118.699316:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699322:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205118.699326:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699333:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205118.699335:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699337:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.699338:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699340:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205118.699342:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205118.699345:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205118.699348:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205118.699350:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205118.699354:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205118.699378:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205930880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205118.699383:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205118.699387:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699393:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205118.699394:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.699396:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205118.699398:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205118.699400:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205118.699403:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931008:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205118.699406:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205118.699408:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205118.699410:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205118.699411:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205118.699415:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205118.699417:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205118.699420:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205118.699423:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205118.699426:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205118.699429:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931008, portal 4 00000100:00000001:0.0:1788205118.699431:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205118.699432:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:0.0:1788205118.699438:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699443:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205118.699446:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205118.699448:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205118.699450:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205931008, offset 0 00000400:00000200:0.0:1788205118.699454:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205118.699458:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.699462:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205118.699464:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205118.699558:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205118.699561:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205118.699564:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205118.699567:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59129:59129:256:4294967295] 192.168.201.151@tcp LPNI seq info [59109:59109:8:4294967295] 00000400:00000200:0.0:1788205118.699571:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205118.699576:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205118.699581:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.699585:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205118.699589:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205118.699592:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205118.699596:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205118.699599:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205118.699602:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205118.699604:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205118.699607:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205118.699611:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205118.699614:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205118.699616:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205118.699618:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205118.699622:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699630:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.699633:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699639:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.699641:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205118.699644:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205118.699647:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205118.699650:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205118.699652:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301246001 00000100:00000001:0.0:1788205118.699654:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.699656:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301246001 00000100:00000001:0.0:1788205118.699658:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301250050 00000100:00000001:0.0:1788205118.699660:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205118.699662:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.699664:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205118.699668:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205118/real 1788205118] req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205118.699675:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205118.699677:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205118.699679:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205118.699681:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205118.699683:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205118.699685:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205118.699688:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205118.699695:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699701:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205118.699703:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699710:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.699712:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205118.699716:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699722:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205118.699726:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699733:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205118.699735:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699737:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205118.699738:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699739:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205118.699742:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205118.699744:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205118.699747:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205118.699749:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205118.699753:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205118.699767:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931008:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205118.699771:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699773:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.699776:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205930880/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699782:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205118.699784:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205118.699786:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205118.699789:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205118.699790:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205118.699792:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205118.699794:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205118.699796:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23c00. 02000000:00000001:0.0:1788205118.699799:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205118.699800:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205118.699802:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:0.0:1788205118.699804:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205118.699805:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699807:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205118.699810:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205931008/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205118.699816:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205118.699817:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205118.699818:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205118.699820:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205118.699821:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205118.699823:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205118.699824:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205118.699826:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22000. 02000000:00000001:0.0:1788205118.699828:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205118.699829:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205118.699830:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:0.0:1788205118.699832:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205118.699834:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699836:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205118.699841:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205118.699842:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205118.699844:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205118.699846:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205118.888531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205118.888537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205118.888543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205118.888544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205118.888547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205118.888549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205118.888552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205118.888553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205118.888613:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205118.888617:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205118.888620:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205119.208519:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205119.208527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205119.208529:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205119.208533:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205119.208537:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205119.208540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205119.208543:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205119.208545:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205119.208548:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205119.656582:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205119.656590:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205119.656592:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205119.656598:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205119.656602:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205119.656605:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205119.656607:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205119.656609:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205119.656613:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205119.720619:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205119.720625:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205119.720626:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205119.720630:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205119.720630:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205119.720633:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205119.720637:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205119.720641:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205119.720645:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205119.720647:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205119.720649:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205119.720652:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205119.720653:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205119.720655:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205119.720658:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205119.720659:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205119.720662:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205119.720665:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205119.912486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205119.912492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205119.912498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205119.912499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205119.912502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205119.912503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205119.912506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205119.912507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205119.912572:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205119.912576:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205119.912579:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205120.232578:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205120.232585:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205120.232588:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205120.232593:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205120.232596:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205120.232600:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.232603:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205120.232605:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.232608:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.680487:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205120.680495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205120.680497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205120.680502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205120.680505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205120.680508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.680510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205120.680512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.680515:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205120.744461:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205120.744470:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205120.744472:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205120.744477:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205120.744481:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205120.744485:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205120.744488:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205120.744492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205120.744495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.744536:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205120.744540:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205120.744541:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205120.744553:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205120.744558:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205120.744561:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.744563:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205120.744564:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.744567:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205120.936541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205120.936548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205120.936553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205120.936554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205120.936557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205120.936558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205120.936561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205120.936563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205120.936645:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205120.936648:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205120.936651:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205121.256478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205121.256484:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205121.256485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205121.256489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205121.256491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205121.256494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205121.256496:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205121.256497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205121.256499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205121.704479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205121.704484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205121.704486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205121.704489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205121.704491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205121.704494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205121.704496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205121.704497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205121.704499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205121.768486:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205121.768494:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205121.768496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205121.768501:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205121.768505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205121.768508:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205121.768511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205121.768515:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205121.768515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205121.768518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205121.768520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205121.768523:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205121.768527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205121.768530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205121.768532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205121.768535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205121.768537:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205121.768538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960435:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205121.960442:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960449:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205121.960452:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205121.960455:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205121.960459:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960462:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205121.960464:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960466:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960469:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205121.960475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205121.960479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205121.960480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205121.960538:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205121.960541:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205121.960544:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205122.280491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205122.280497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205122.280499:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205122.280503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205122.280505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205122.280509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.280511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205122.280517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.280520:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205122.728472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205122.728481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205122.728482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205122.728486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205122.728489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205122.728491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.728493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205122.728494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.728497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205122.792467:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205122.792474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205122.792475:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205122.792479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788205122.792480:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205122.792482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205122.792485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205122.792486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205122.792490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205122.792491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205122.792493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205122.792494:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205122.792497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205122.792497:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205122.792502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205122.792502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205122.792506:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205122.792509:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205122.984533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205122.984540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205122.984546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205122.984547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205122.984551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205122.984552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205122.984555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205122.984556:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205122.984614:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205122.984618:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205122.984621:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205123.304481:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301250050, at 4301250560 00000100:00000040:1.0:1788205123.304481:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301250050, at 4301250560 00000100:00000001:3.0:1788205123.304482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205123.304486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205123.304490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205123.304494:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000010:1.0:1788205123.304496:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff97e0. 00000100:00100000:3.0:1788205123.304496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205123.304500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205123.304503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.304506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205123.304508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.304511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205123.752469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205123.752474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205123.752476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205123.752479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205123.752481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205123.752484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.752486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205123.752488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.752490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205123.816554:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205123.816562:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205123.816564:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205123.816569:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205123.816573:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205123.816576:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.816578:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205123.816582:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.816584:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.816696:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205123.816699:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205123.816701:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205123.816704:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205123.816706:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205123.816709:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205123.816715:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205123.816718:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205123.816721:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205123.816722:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205123.816727:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205123.816729:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205123.816731:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.816735:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205123.816738:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6578 00000400:00000200:1.0:1788205123.816748:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205123.816758:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205123.816762:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205123.816765:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816769:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205123.816771:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816774:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205123.816777:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205123.816779:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205123.816781:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205123.816785:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205123.816787:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205123.816790:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205123.816793:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816794:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205123.816796:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816798:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205123.816801:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205123.816804:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816806:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205123.816809:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205123.816812:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205123.816814:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.816816:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205123.816819:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205123.816827:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788205123.816831:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205123.816834:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205123.816835:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.816837:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205123.816839:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205123.816843:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205123.816850:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788205123.816860:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205123.816868:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205123.816878:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205123.816883:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205123.816887:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205123.816905:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205123.816908:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205123.816911:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205123.816912:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205123.816914:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205123.816916:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205123.816918:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6578 00000400:00000200:1.0:1788205123.816923:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205123.816927:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205123.816929:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205123.816932:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205123.816934:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205123.816934:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205123.816940:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205123.816940:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205123.816944:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205123.816945:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205123.816949:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205123.816950:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205123.816952:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:0.0:1788205123.816954:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205123.816955:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205123.816960:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205123.816965:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788205123.816971:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205123.816975:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205123.816975:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205123.816980:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816981:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205123.816983:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205123.816985:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205123.816987:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205123.816989:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205123.816990:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205123.816993:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205123.816995:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205123.816997:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.816999:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205123.817000:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1788205123.817005:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788205123.817006:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205123.817011:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205123.817012:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205931136:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205123.817014:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205123.817020:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000100:00000001:0.0:1788205123.817021:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205123.817027:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205123.817029:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.817031:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1788205123.817033:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000010:0.0:1788205123.817037:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:1.0:1788205123.817038:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205123.817040:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.817042:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817043:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205123.817046:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205123.817050:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.817054:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:1.0:1788205123.817057:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000400:00000010:0.0:1788205123.817060:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000100:00000001:1.0:1788205123.817062:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205123.817069:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00100000:1.0:1788205123.817070:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1788205123.817078:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931136, portal 25 00000100:00000001:1.0:1788205123.817079:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205123.817083:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205123.817084:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817086:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:1.0:1788205123.817090:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205123.817097:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205123.817103:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205123.817108:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205123.817112:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788205123.817113:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205123.817117:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:0.0:1788205123.817118:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788205123.817120:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205123.817123:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205123.817126:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6578 00000400:00000200:1.0:1788205123.817130:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205123.817133:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000400:00000200:1.0:1788205123.817134:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000200:0.0:1788205123.817139:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205931136, offset 0 00000100:00080000:1.0:1788205123.817140:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205123.817146:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817148:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205123.817151:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:1.0:1788205123.817154:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817158:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205123.817160:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1788205123.817162:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788205123.817163:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205123.817167:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205123.817169:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:0.0:1788205123.817173:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:1.0:1788205123.817173:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205123.817178:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205123.817182:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1788205123.817183:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817187:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205123.817190:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:1.0:1788205123.817193:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817196:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205123.817197:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1788205123.817200:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:0.0:1788205123.817203:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:1.0:1788205123.817207:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817211:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.817213:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59130:59130:256:4294967295] 192.168.201.151@tcp LPNI seq info [59110:59110:8:4294967295] 02000000:00000001:1.0:1788205123.817214:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205123.817217:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205123.817222:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817225:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1788205123.817226:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:1.0:1788205123.817227:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205123.817231:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817232:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205123.817234:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788205123.817238:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205123.817241:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205123.817244:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.817245:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788205123.817246:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205123.817249:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205123.817251:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205123.817255:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000800:00000200:0.0:1788205123.817259:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817260:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205123.817268:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000100:00100000:1.0:1788205123.817268:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1788205123.817282:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205123.817289:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1788205123.817290:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205123.817294:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788205123.817299:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788205123.817301:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788205123.817305:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205123.817317:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817318:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205123.817321:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000100:00080000:1.0:1788205123.817322:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205123.817327:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205123.817328:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205123.817329:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205123.817330:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1788205123.817332:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1788205123.817333:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6578 00000400:00000200:1.0:1788205123.817338:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205123.817342:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:0.0:1788205123.817344:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1788205123.817345:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205123.817351:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205123.817354:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000100:00000001:1.0:1788205123.817355:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1788205123.817359:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000040:1.0:1788205123.817360:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205123.817382:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205123.817403:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205123.817410:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205123.817416:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205123.817418:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:0.0:1788205123.817420:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788205123.817420:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205123.817425:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000040:0.0:1788205123.817426:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205123.817428:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205123.817433:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205123.817441:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205123.817444:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817445:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205123.817448:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1788205123.817449:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817452:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205123.817454:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:0.0:1788205123.817456:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205123.817457:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205123.817462:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205123.817463:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1788205123.817466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788205123.817466:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205123.817471:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817472:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301255170 00000100:00000001:1.0:1788205123.817472:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205123.817475:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205123.817478:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817479:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205123.817480:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000010:1.0:1788205123.817482:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788205123.817485:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205123.817487:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205123.817488:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817489:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205123.817489:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205123.817491:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205123.817494:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205123.817497:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:0.0:1788205123.817499:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301255170 00000100:00000001:1.0:1788205123.817500:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205123.817505:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9120. 00000100:00100000:1.0:1788205123.817506:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205123.817512:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205123.817513:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205123.817517:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817519:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205123.817520:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817526:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205123.817528:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817531:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788205123.817532:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6588) 00000100:00100000:0.0:1788205123.817540:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205123/real 1788205123] req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205123.817549:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205123.817551:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205123.817554:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205123.817556:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205123.817558:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205123.817560:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205123.817563:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205123.817572:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817579:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205123.817580:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817582:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.817584:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205123.817588:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817607:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205123.817612:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817618:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205123.817621:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817622:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.817623:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817626:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205123.817628:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205123.817630:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205123.817634:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205123.817636:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205123.817640:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205123.817648:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205931136:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205123.817654:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817656:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.817659:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205931136/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817666:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205123.817668:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205123.817670:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205123.817672:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205123.817674:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205123.817676:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205123.817678:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205123.817680:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:0.0:1788205123.817683:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205123.817684:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205123.817685:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:0.0:1788205123.817688:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205123.817689:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817691:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.817697:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205123.817699:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.817700:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205123.817702:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205123.817703:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205123.817706:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.817955:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205123.817956:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.817958:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205123.817961:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205123.817962:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205123.817966:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931264:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205123.817968:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205123.817970:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205123.817971:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.817973:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205123.817986:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205123.817989:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.817993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205123.817995:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205123.817999:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205123.818002:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931264, portal 10 00000100:00000001:0.0:1788205123.818003:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205123.818005:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:0.0:1788205123.818011:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818017:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205123.818019:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205123.818022:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205123.818024:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205931264, offset 0 00000400:00000200:0.0:1788205123.818027:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818032:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818035:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205123.818039:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205123.818042:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205123.818044:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205123.818046:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818049:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59131:59131:256:4294967295] 192.168.201.151@tcp LPNI seq info [59111:59111:8:4294967295] 00000400:00000200:0.0:1788205123.818053:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818058:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205123.818063:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818066:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205123.818070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205123.818073:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788205123.818077:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205123.818079:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205123.818083:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818084:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205123.818088:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205123.818091:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205123.818094:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205123.818096:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205123.818098:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205123.818101:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818110:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.818112:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818179:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818181:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205123.818185:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205123.818189:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205123.818192:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205123.818194:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301251227 00000100:00000001:0.0:1788205123.818196:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301251227 00000100:00000001:0.0:1788205123.818200:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818202:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205123.818204:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818206:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205123.818210:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205123/real 1788205123] req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205123.818218:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205123.818219:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205123.818221:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205123.818223:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205123.818225:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205123.818227:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205123.818230:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205123.818238:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818243:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205123.818245:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818246:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818248:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205123.818251:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818257:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205123.818261:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818267:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205123.818269:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818271:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818272:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818274:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205123.818276:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205123.818278:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205123.818281:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205123.818283:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205123.818287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205123.818294:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931264:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205123.818297:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205123.818300:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818311:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205123.818313:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.818315:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205123.818316:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205123.818318:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205123.818321:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205123.818323:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205123.818325:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205123.818327:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.818328:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205123.818331:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205123.818333:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.818336:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205123.818338:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205123.818341:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205123.818344:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931392, portal 4 00000100:00000001:0.0:1788205123.818346:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205123.818347:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:0.0:1788205123.818353:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818359:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205123.818361:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205123.818381:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205123.818384:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205931392, offset 0 00000400:00000200:0.0:1788205123.818400:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818408:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205123.818410:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205123.818413:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205123.818415:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205123.818417:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818420:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59132:59132:256:4294967295] 192.168.201.151@tcp LPNI seq info [59112:59112:8:4294967295] 00000400:00000200:0.0:1788205123.818424:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818430:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205123.818434:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818437:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205123.818441:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205123.818444:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788205123.818447:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205123.818450:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205123.818454:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818456:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205123.818459:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205123.818462:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205123.818465:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205123.818467:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205123.818469:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205123.818472:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818479:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.818482:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818489:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818490:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205123.818493:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205123.818495:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205123.818499:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205123.818501:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301251114 00000100:00000001:0.0:1788205123.818503:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818505:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301251114 00000100:00000001:0.0:1788205123.818507:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818509:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205123.818512:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818513:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205123.818517:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205123/real 1788205123] req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205123.818525:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205123.818526:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205123.818528:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205123.818530:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205123.818532:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205123.818534:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205123.818537:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205123.818545:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818551:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205123.818553:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818554:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818555:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205123.818559:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818565:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205123.818568:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818574:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205123.818576:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818577:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818579:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818581:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205123.818583:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205123.818585:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205123.818600:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205123.818602:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205123.818607:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205123.818614:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205123.818618:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205123.818621:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818627:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205123.818628:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.818630:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205123.818632:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205123.818634:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205123.818636:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931520:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205123.818639:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205123.818641:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205123.818642:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205123.818644:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205123.818647:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205123.818649:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205123.818652:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205123.818654:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205123.818657:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205123.818660:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931520, portal 4 00000100:00000001:0.0:1788205123.818661:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205123.818663:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00000040:0.0:1788205123.818668:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818674:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205123.818677:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205123.818679:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205123.818681:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205931520, offset 0 00000400:00000200:0.0:1788205123.818684:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818688:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818692:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205123.818694:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205123.818697:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205123.818699:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205123.818701:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205123.818703:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59133:59133:256:4294967295] 192.168.201.151@tcp LPNI seq info [59113:59113:8:4294967295] 00000400:00000200:0.0:1788205123.818707:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205123.818787:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205123.818792:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818796:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205123.818800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205123.818802:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788205123.818806:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205123.818808:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205123.818812:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205123.818814:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205123.818817:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205123.818820:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205123.818822:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205123.818824:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205123.818826:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205123.818830:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818838:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.818841:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818847:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818848:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205123.818851:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205123.818854:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205123.818857:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205123.818859:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301251120 00000100:00000001:0.0:1788205123.818861:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818863:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301251120 00000100:00000001:0.0:1788205123.818865:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301255170 00000100:00000001:0.0:1788205123.818866:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205123.818868:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.818870:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205123.818873:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205123/real 1788205123] req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205123.818881:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205123.818882:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205123.818884:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205123.818886:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205123.818888:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205123.818889:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205123.818892:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205123.818899:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818905:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205123.818907:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818908:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818910:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205123.818913:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818919:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205123.818923:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818930:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205123.818932:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818934:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205123.818935:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818936:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205123.818938:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205123.818941:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205123.818944:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205123.818946:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205123.818949:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205123.818957:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931520:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205123.818960:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818962:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.818965:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205931264/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.818971:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205123.818972:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205123.818974:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205123.818977:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205123.818978:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205123.818979:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205123.818981:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205123.818983:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:0.0:1788205123.818985:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205123.818987:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205123.818988:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:0.0:1788205123.818990:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205123.818992:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.818994:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.818997:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205931392/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.819003:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205123.819004:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205123.819005:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205123.819007:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205123.819008:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205123.819009:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205123.819011:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205123.819012:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:0.0:1788205123.819014:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205123.819015:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205123.819016:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:0.0:1788205123.819017:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205123.819019:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.819020:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205123.819023:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205931520/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205123.819029:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205123.819030:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205123.819031:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205123.819033:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205123.819034:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205123.819035:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205123.819037:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205123.819039:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:0.0:1788205123.819041:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205123.819042:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205123.819043:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:0.0:1788205123.819045:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205123.819046:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.819047:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205123.819051:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205123.819053:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205123.819054:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205123.819056:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205124.008519:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205124.008523:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205124.008527:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008533:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205124.008536:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008539:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008542:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205124.008548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205124.008554:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205124.008556:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205124.008557:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008559:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205124.008561:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008562:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008564:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205124.008569:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205124.008570:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205124.008572:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008574:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205124.008575:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008576:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205124.008578:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205124.008651:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205124.008655:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205124.008658:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205124.328499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205124.328507:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205124.328509:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205124.328515:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205124.328518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205124.328522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.328525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205124.328528:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.328530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205124.776433:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205124.776439:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205124.776440:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205124.776444:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205124.776447:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205124.776449:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.776451:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205124.776453:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.776455:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205124.840489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205124.840496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205124.840497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205124.840498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205124.840502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205124.840505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205124.840505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205124.840511:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205124.840512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205124.840516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205124.840517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205124.840522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.840522:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205124.840526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.840530:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205124.840533:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205124.840537:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205124.840540:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205125.032509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205125.032515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205125.032520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205125.032521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205125.032524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205125.032525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205125.032527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205125.032529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205125.032582:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205125.032584:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205125.032587:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205125.352503:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205125.352510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205125.352511:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205125.352516:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205125.352519:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205125.352524:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.352526:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205125.352528:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.352531:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205125.800464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205125.800470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205125.800472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205125.800475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205125.800478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205125.800481:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.800483:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205125.800484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.800487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205125.864468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205125.864475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205125.864477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205125.864482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205125.864485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205125.864489:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.864491:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205125.864495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.864498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.864507:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205125.864512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205125.864513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205125.864517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205125.864520:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205125.864523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205125.864525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205125.864527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205125.864529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205126.056474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205126.056481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205126.056488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205126.056489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205126.056493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205126.056494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205126.056498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205126.056499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205126.056550:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205126.056554:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205126.056557:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205126.376459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205126.376467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205126.376469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205126.376475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205126.376478:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205126.376482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.376484:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205126.376487:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.376489:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205126.824484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205126.824491:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205126.824493:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205126.824498:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205126.824501:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205126.824505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.824507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205126.824509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.824512:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205126.888483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205126.888491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205126.888493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205126.888494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205126.888500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205126.888501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205126.888502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205126.888508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205126.888508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205126.888513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205126.888514:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205126.888516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.888519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205126.888523:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.888527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205126.888528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205126.888531:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205126.888534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205127.080507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205127.080512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205127.080516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205127.080517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205127.080520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205127.080521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205127.080523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205127.080523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205127.080558:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205127.080562:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205127.080565:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205127.400470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205127.400476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205127.400477:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205127.400481:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205127.400483:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205127.400486:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.400488:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205127.400490:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.400492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205127.848520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205127.848531:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205127.848534:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205127.848541:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205127.848546:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205127.848550:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.848552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205127.848555:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.848558:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205127.912468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205127.912480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205127.912484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205127.912492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205127.912497:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205127.912502:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.912506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205127.912510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.912513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205127.912555:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205127.912563:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205127.912566:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205127.912573:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205127.912578:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205127.912581:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205127.912584:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205127.912586:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205127.912589:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205128.104493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205128.104500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205128.104506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205128.104508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205128.104512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205128.104513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205128.104516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205128.104518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205128.104578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205128.104581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205128.104583:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205128.424497:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301255170, at 4301255680 00000100:00000001:3.0:1788205128.424510:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205128.424519:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788205128.424520:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000001:3.0:1788205128.424523:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.424531:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205128.424535:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205128.424540:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.424544:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.424548:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.424551:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205128.425381:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301255170, at 4301255681 00000100:00000010:1.0:1788205128.425389:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9120. 00000100:00000001:3.0:1788205128.872518:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205128.872528:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205128.872531:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.872537:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205128.872542:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205128.872547:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.872550:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.872554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.872558:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205128.936471:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205128.936478:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205128.936481:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205128.936484:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205128.936488:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205128.936494:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6583 00000100:00000001:3.0:1788205128.936494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205128.936502:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205128.936506:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205128.936513:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:3.0:1788205128.936517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000400:00000200:0.0:1788205128.936519:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205128.936525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205128.936526:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205128.936532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205128.936534:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205128.936538:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205128.936540:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205128.936544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205128.936545:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936551:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205128.936551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205128.936559:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205128.936562:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205128.936565:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205128.936569:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205128.936572:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205128.936574:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205128.936578:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936579:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205128.936581:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936582:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205128.936584:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205128.936587:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936589:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.936591:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205128.936599:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 02000000:00000001:0.0:1788205128.936600:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205128.936605:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1788205128.936607:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.936609:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936612:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205128.936615:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.936624:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000010:0.0:1788205128.936625:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205128.936628:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788205128.936632:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 00000100:00000001:3.0:1788205128.936633:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205128.936638:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205128.936639:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.936642:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.936645:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.936650:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936655:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205128.936658:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205128.936662:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205128.936671:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788205128.936678:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205128.936687:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205128.936697:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205128.936701:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205128.936706:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205128.936732:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205128.936734:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205128.936737:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205128.936738:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205128.936739:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205128.936740:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205128.936742:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6583 00000400:00000200:0.0:1788205128.936745:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205128.936747:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205128.936749:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205128.936752:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936755:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205128.936757:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936758:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205128.936760:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205128.936761:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205128.936762:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205128.936763:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788205128.936765:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205128.936766:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205128.936770:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788205128.936772:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788205128.936773:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.936776:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205128.936777:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205128.936777:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205128.936782:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205128.936784:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205128.936790:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:3.0:1788205128.936793:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205128.936794:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.936800:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205128.936806:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205128.936809:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936811:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205128.936816:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205128.936818:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1788205128.936822:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.936825:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936827:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205128.936833:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 02000000:00000001:3.0:1788205128.936833:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1788205128.936836:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205128.936839:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205128.936840:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205128.936844:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936853:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205128.936853:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.936859:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205128.936861:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.936866:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205931648:192.168.201.151@tcp:250:kworker.0 02000000:00000010:0.0:1788205128.936867:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:0.0:1788205128.936874:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.936879:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1788205128.936882:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205128.936886:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205128.936893:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.936894:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205128.936899:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788205128.936900:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205128.936909:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788205128.936911:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205128.936917:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.936929:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205128.936934:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788205128.936939:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205128.936945:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788205128.936945:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00080000:0.0:1788205128.936954:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205128.936956:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205128.936957:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788205128.936959:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931648, portal 25 00000100:00000001:0.0:1788205128.936962:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205128.936964:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788205128.936969:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6583 00000100:00000001:3.0:1788205128.936969:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000400:00000200:0.0:1788205128.936979:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:3.0:1788205128.936984:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205128.936989:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205128.936995:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205128.937002:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937008:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205128.937012:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937020:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205128.937022:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788205128.937023:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:3.0:1788205128.937031:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:0.0:1788205128.937032:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205128.937035:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:3.0:1788205128.937037:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000020:00000040:0.0:1788205128.937038:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205128.937042:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205128.937046:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:3.0:1788205128.937046:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205931648, offset 0 00000100:00000040:0.0:1788205128.937059:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937065:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205128.937067:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788205128.937073:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937077:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205128.937080:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788205128.937082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788205128.937087:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937091:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.937095:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205128.937096:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000001:0.0:1788205128.937103:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205128.937108:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1788205128.937110:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.937115:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205128.937118:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000008:00100000:0.0:1788205128.937120:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205128.937125:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.937126:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788205128.937130:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.937134:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000010:0.0:1788205128.937138:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000400:00000200:3.0:1788205128.937144:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59134:59134:256:4294967295] 192.168.201.151@tcp LPNI seq info [59114:59114:8:4294967295] 00000020:00000040:0.0:1788205128.937145:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788205128.937153:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788205128.937159:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205128.937161:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.937163:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205128.937167:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205128.937172:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205128.937182:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca000. 00000400:00000200:3.0:1788205128.937182:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788205128.937190:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205128.937198:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205128.937201:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937214:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205128.937216:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205128.937218:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:0.0:1788205128.937227:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788205128.937234:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205128.937241:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1788205128.937246:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205128.937252:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788205128.937253:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205128.937263:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00080000:0.0:1788205128.937264:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205128.937268:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205128.937273:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205128.937277:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:3.0:1788205128.937278:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00080000:0.0:1788205128.937283:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6583 00000800:00000010:3.0:1788205128.937285:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:0.0:1788205128.937292:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205128.937314:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205128.937316:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205128.937319:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937322:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205128.937323:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1788205128.937324:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937331:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205128.937339:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788205128.937339:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000040:0.0:1788205128.937344:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205128.937347:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205128.937349:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205128.937351:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:3.0:1788205128.937352:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000020:00000001:0.0:1788205128.937353:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000010:3.0:1788205128.937358:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000040:0.0:1788205128.937359:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937394:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205128.937396:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937397:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205128.937399:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205128.937401:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205128.937402:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205128.937404:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205128.937406:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205128.937409:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.937411:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205128.937412:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1788205128.937413:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205128.937417:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.937418:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205128.937423:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788205128.937425:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205128.937428:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205128.937435:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205128.937438:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205128.937444:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205128.937446:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205128.937450:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205128.937455:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:3.0:1788205128.937457:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205128.937458:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205128.937471:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205128.937472:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205128.937541:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205128.937541:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937550:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205128.937553:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937559:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205128.937562:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205128.937569:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205128.937575:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301260290 00000100:00000010:3.0:1788205128.937584:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4c60. 00000100:00000001:0.0:1788205128.937587:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937594:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788205128.937595:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6593) 00000100:00000001:3.0:1788205128.937611:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301260290 00000100:00000010:3.0:1788205128.937617:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f44e0. 00000100:00000001:3.0:1788205128.937621:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205128.937627:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205128.937631:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937634:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205128.937640:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205128/real 1788205128] req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205128.937650:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205128.937652:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205128.937654:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205128.937657:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205128.937658:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205128.937661:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205128.937664:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205128.937672:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937679:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205128.937681:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937683:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.937685:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205128.937689:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937696:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205128.937700:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937708:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205128.937711:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937712:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.937713:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937716:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205128.937718:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205128.937721:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205128.937725:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205128.937727:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205128.937732:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205128.937739:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205931648:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205128.937743:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937745:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.937748:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205931648/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937754:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205128.937756:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205128.937759:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205128.937762:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205128.937765:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205128.937767:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205128.937769:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205128.937772:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:3.0:1788205128.937776:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205128.937778:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205128.937780:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205128.937783:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205128.937784:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937787:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.937795:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205128.937797:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937799:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205128.937801:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205128.937802:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205128.937805:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937812:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.937813:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.937815:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205128.937818:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205128.937820:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.937823:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931776:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205128.937825:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205128.937827:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205128.937829:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.937831:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205128.937834:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205128.937835:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.937838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205128.937841:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205128.937844:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205128.937846:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931776, portal 10 00000100:00000001:3.0:1788205128.937848:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205128.937849:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205128.937855:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937861:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205128.937863:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205128.937865:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205128.937867:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205931776, offset 0 00000400:00000200:3.0:1788205128.937871:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205128.937874:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.937877:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205128.937880:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205128.937883:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205128.937885:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205128.937887:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205128.937890:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59135:59135:256:4294967295] 192.168.201.151@tcp LPNI seq info [59115:59115:8:4294967295] 00000400:00000200:3.0:1788205128.937894:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.937900:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205128.937904:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.937906:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205128.937910:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205128.937913:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788205128.937917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205128.937920:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788205128.937923:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.937925:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205128.937928:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205128.937931:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205128.937934:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205128.937936:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205128.937938:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205128.937940:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937948:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.937951:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.937963:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937964:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205128.937967:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205128.937970:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205128.937974:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205128.937976:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301256347 00000100:00000001:3.0:1788205128.937979:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.937981:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301256347 00000100:00000001:3.0:1788205128.937983:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.937985:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205128.937987:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.937988:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205128.937992:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205128/real 1788205128] req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205128.938001:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205128.938002:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205128.938003:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205128.938006:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205128.938007:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205128.938009:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205128.938012:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205128.938020:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938026:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205128.938027:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938029:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938031:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205128.938034:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938042:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205128.938046:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938052:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205128.938054:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938056:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938057:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938058:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205128.938060:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205128.938062:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205128.938065:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205128.938067:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205128.938070:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205128.938078:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931776:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205128.938082:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205128.938085:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938092:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.938093:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.938095:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205128.938097:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205128.938099:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.938101:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205128.938104:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205128.938106:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205128.938108:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.938109:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205128.938112:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205128.938114:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.938117:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205128.938119:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205128.938122:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205128.938124:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205931904, portal 4 00000100:00000001:3.0:1788205128.938126:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205128.938127:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205128.938132:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938138:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205128.938140:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205128.938142:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205128.938144:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205931904, offset 0 00000400:00000200:3.0:1788205128.938148:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205128.938151:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.938154:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205128.938157:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205128.938159:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205128.938162:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205128.938164:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205128.938167:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59136:59136:256:4294967295] 192.168.201.151@tcp LPNI seq info [59116:59116:8:4294967295] 00000400:00000200:3.0:1788205128.938172:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.938177:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205128.938181:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.938184:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205128.938187:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205128.938190:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788205128.938194:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205128.938196:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788205128.938200:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.938202:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205128.938205:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205128.938208:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205128.938211:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205128.938213:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205128.938215:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205128.938218:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938226:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.938229:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938236:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.938238:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205128.938241:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205128.938244:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205128.938247:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205128.938249:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301256233 00000100:00000001:3.0:1788205128.938252:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.938254:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301256233 00000100:00000001:3.0:1788205128.938256:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.938258:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205128.938260:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.938262:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205128.938265:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205128/real 1788205128] req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205128.938282:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205128.938284:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205128.938289:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205128.938291:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205128.938293:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205128.938295:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205128.938308:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205128.938316:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938321:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205128.938322:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938323:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938324:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205128.938326:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938331:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205128.938333:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938336:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205128.938337:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938338:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938339:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938340:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205128.938342:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205128.938343:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205128.938345:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205128.938346:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205128.938348:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205128.938352:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205931904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205128.938355:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205128.938357:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938362:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.938379:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.938381:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205128.938382:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205128.938384:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205128.938386:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932032:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205128.938388:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205128.938389:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205128.938391:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205128.938392:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205128.938394:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205128.938396:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205128.938398:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205128.938399:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205128.938401:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205128.938403:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932032, portal 4 00000100:00000001:3.0:1788205128.938404:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205128.938405:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205128.938409:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938413:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205128.938414:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205128.938416:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205128.938417:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205932032, offset 0 00000400:00000200:3.0:1788205128.938419:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205128.938421:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.938423:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205128.938425:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205128.938427:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205128.938428:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205128.938430:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205128.938432:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59137:59137:256:4294967295] 192.168.201.151@tcp LPNI seq info [59117:59117:8:4294967295] 00000400:00000200:3.0:1788205128.938435:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205128.938439:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205128.938442:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.938444:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205128.938446:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205128.938448:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788205128.938450:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205128.938452:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788205128.938454:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205128.938455:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205128.938457:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205128.938460:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205128.938461:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205128.938462:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205128.938464:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205128.938466:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938470:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.938472:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938476:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.938477:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205128.938478:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205128.938480:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205128.938482:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205128.938483:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301256240 00000100:00000001:3.0:1788205128.938484:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.938485:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301256240 00000100:00000001:3.0:1788205128.938486:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301260290 00000100:00000001:3.0:1788205128.938488:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205128.938489:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.938491:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205128.938493:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205128/real 1788205128] req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205128.938497:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205128.938498:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205128.938499:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205128.938500:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205128.938501:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205128.938502:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205128.938504:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205128.938509:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938512:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205128.938513:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938514:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938514:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205128.938516:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938520:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205128.938521:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938525:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205128.938526:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938527:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205128.938529:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938530:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205128.938531:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205128.938533:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205128.938535:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205128.938537:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205128.938539:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205128.938543:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932032:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205128.938545:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938546:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.938548:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205931776/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938551:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205128.938552:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205128.938553:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22800. 02000000:00000001:3.0:1788205128.938554:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205128.938555:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205128.938556:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205128.938557:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205128.938558:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:3.0:1788205128.938559:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205128.938560:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205128.938561:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205128.938563:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205128.938564:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938564:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.938566:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205931904/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938569:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205128.938570:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205128.938571:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c20400. 02000000:00000001:3.0:1788205128.938572:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205128.938573:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205128.938573:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205128.938574:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205128.938576:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca000. 02000000:00000001:3.0:1788205128.938577:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205128.938578:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205128.938579:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205128.938580:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205128.938581:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938582:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205128.938583:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205932032/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205128.938586:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205128.938587:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205128.938588:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21400. 02000000:00000001:3.0:1788205128.938589:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205128.938589:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205128.938590:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205128.938591:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205128.938592:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:3.0:1788205128.938593:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205128.938593:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205128.938594:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205128.938595:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205128.938596:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938597:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.938599:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205128.938600:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.938601:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205128.938602:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205129.128533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205129.128543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205129.128551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205129.128552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205129.128554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205129.128555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205129.128557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205129.128558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205129.128633:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205129.128638:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205129.128642:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205129.448457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205129.448462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205129.448464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205129.448467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205129.448470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205129.448473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.448475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205129.448477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.448479:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205129.896473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205129.896479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205129.896481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205129.896486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205129.896489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205129.896493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.896495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205129.896497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.896500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205129.960504:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205129.960504:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205129.960510:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205129.960510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205129.960513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205129.960513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205129.960517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205129.960517:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205129.960521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205129.960521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205129.960525:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205129.960525:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205129.960527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205129.960530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205129.960532:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205129.960534:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205129.960536:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205129.960538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205130.152491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205130.152498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205130.152504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205130.152505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205130.152509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205130.152510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205130.152514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205130.152515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205130.152580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205130.152584:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205130.152588:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205130.472469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205130.472475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205130.472476:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.472480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205130.472484:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205130.472488:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.472490:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.472493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.472496:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.920461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205130.920472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205130.920474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.920479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205130.920482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205130.920486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.920489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.920491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.920494:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.984481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205130.984487:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205130.984490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.984494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788205130.984495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205130.984499:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205130.984501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205130.984503:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.984507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.984508:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205130.984511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.984513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205130.984516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.984517:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205130.984521:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.984521:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.984524:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.984527:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205131.176476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176481:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205131.176485:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205131.176487:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205131.176490:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176492:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205131.176493:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176494:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176496:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205131.176499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205131.176502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205131.176503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205131.176571:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205131.176575:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205131.176579:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205131.496498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205131.496506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205131.496507:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205131.496513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205131.496517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205131.496521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.496523:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205131.496527:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.496530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205131.944479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205131.944485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205131.944487:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205131.944490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205131.944492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205131.944495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.944497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205131.944498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.944500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205132.008468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205132.008468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205132.008475:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205132.008475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205132.008477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205132.008477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205132.008482:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205132.008482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205132.008486:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205132.008486:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205132.008490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.008490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.008492:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.008495:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205132.008498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.008498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205132.008499:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205132.008501:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205132.200482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205132.200489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205132.200496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205132.200497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205132.200501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205132.200502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205132.200505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205132.200506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205132.200578:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205132.200582:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205132.200584:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205132.520485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205132.520492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205132.520494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205132.520499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205132.520503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205132.520507:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.520509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.520512:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.520515:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205132.968457:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205132.968467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205132.968470:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205132.968478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205132.968482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205132.968486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.968494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.968496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.968499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205133.032431:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205133.032440:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205133.032442:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205133.032448:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205133.032452:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205133.032456:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.032459:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205133.032461:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.032464:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205133.033218:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205133.033235:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205133.033238:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205133.033243:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205133.033246:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205133.033250:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205133.033252:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205133.033255:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205133.033258:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224614:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205133.224625:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205133.224628:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205133.224632:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224638:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205133.224641:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224643:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224646:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224650:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205133.224651:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224655:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205133.224657:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205133.224658:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205133.224659:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224660:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205133.224662:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224664:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224665:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224669:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205133.224670:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205133.224672:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205133.224673:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224675:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205133.224677:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224679:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205133.224681:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205133.224785:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205133.224789:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205133.224792:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205133.544495:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301260290, at 4301260800 00000100:00000001:3.0:1788205133.544496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205133.544503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205133.544506:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205133.544512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205133.544514:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4c60. 00000100:00000001:3.0:1788205133.544516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205133.544522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.544588:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205133.544591:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.544595:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205133.544680:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301260290, at 4301260800 00000100:00000010:1.0:1788205133.544686:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f44e0. 00000100:00000001:3.0:1788205133.992491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205133.992499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205133.992502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205133.992507:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205133.992511:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205133.992576:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.992580:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205133.992583:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.992586:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205134.056472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205134.056481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205134.056484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205134.056490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205134.056495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205134.056500:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205134.056503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205134.056506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205134.056510:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205134.056545:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205134.056551:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205134.056554:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205134.056558:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205134.056563:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205134.056573:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6588 00000400:00000200:3.0:1788205134.056586:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205134.056592:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000400:00000200:3.0:1788205134.056596:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788205134.056602:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788205134.056603:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788205134.056609:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.056611:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056619:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788205134.056620:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000040:3.0:1788205134.056623:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056627:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205134.056629:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788205134.056632:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205134.056635:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:2.0:1788205134.056636:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205134.056638:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205134.056644:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000100:00000001:2.0:1788205134.056646:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788205134.056646:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788205134.056653:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205134.056654:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000001:2.0:1788205134.056661:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.056667:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056672:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.056674:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056676:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205134.056680:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205134.056684:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056687:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205134.056691:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205134.056697:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205134.056706:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.056709:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205134.056712:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205134.056723:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0380. 00000020:00000040:3.0:1788205134.056726:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205134.056730:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205134.056732:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.056735:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205134.056737:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205134.056742:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205134.056749:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21400. 00000100:00000001:3.0:1788205134.056756:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.056765:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205134.056777:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205134.056782:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.056788:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2501e67] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205134.056818:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205134.056821:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205134.056825:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205134.056826:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205134.056828:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205134.056832:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205134.056846:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6588 00000400:00000200:3.0:1788205134.056850:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205134.056853:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205134.056855:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205134.056858:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056860:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.056861:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056863:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205134.056866:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205134.056867:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205134.056868:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205134.056870:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205134.056872:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205134.056874:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205134.056877:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056879:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.056880:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056882:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205134.056884:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788205134.056886:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056888:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205134.056890:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205134.056896:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205134.056898:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.056900:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205134.056906:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205134.056909:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.056911:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205134.056917:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3100. 00000020:00000040:3.0:1788205134.056919:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205134.056922:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205134.056923:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.056924:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205134.056926:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205134.056927:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205134.056931:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c20400. 00000100:00000001:3.0:1788205134.056934:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.056940:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205134.056948:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205134.056950:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.056954:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e74a292a] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205134.056962:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205134.056964:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205134.056968:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205134.056970:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205134.056972:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205134.056979:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205134.056982:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6588 00000400:00000200:3.0:1788205134.056986:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205134.056989:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205134.056990:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205134.056992:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056994:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.056995:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.056996:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205134.056997:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205134.056998:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205134.056999:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205134.057000:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788205134.057002:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205134.057004:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788205134.057008:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057010:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.057012:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057013:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205134.057016:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205134.057019:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057021:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205134.057023:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205134.057026:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205134.057028:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057030:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205134.057032:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205134.057034:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057037:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205134.057040:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2d80. 00000020:00000040:3.0:1788205134.057042:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205134.057044:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205134.057045:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057046:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205134.057047:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205134.057048:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205134.057051:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22800. 00000100:00000001:3.0:1788205134.057053:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.057057:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205134.057062:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205134.057064:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.057067:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001291af88] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205134.057072:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205134.057074:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205134.057076:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205134.057078:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205134.057078:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205134.057080:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205134.057081:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6588 00000400:00000200:3.0:1788205134.057085:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205134.057088:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205134.057090:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205134.057093:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057095:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.057096:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057098:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205134.057100:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205134.057102:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205134.057104:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205134.057106:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205134.057108:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205134.057110:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205134.057113:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057115:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205134.057117:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057119:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205134.057121:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205134.057124:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205134.057127:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205134.057129:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205134.057131:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205134.057134:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057136:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205134.057138:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788205134.057139:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057140:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205134.057143:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce3480. 00000020:00000040:3.0:1788205134.057145:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205134.057147:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205134.057148:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.057150:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205134.057151:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205134.057153:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205134.057156:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23000. 00000100:00000001:3.0:1788205134.057160:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.057164:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205134.057171:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205134.057173:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.057177:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133b1fcc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205134.057186:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205134.057191:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6599) 00000100:00000001:0.0:1788205134.057266:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205134.057271:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205134.057273:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205134.057281:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057291:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205134.057292:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.057296:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205134.057298:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205134.057300:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205134.057302:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205932160:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205134.057306:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205134.057310:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205134.057313:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.057315:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205134.057319:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205134.057320:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205134.057325:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205134.057329:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205134.057338:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205134.057346:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932160, portal 25 00000100:00000001:0.0:1788205134.057349:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205134.057351:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300822400 : -113511408729216 : ffff98c310ce0380) 00000100:00000040:0.0:1788205134.057357:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057387:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205134.057391:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205134.057394:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205134.057398:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205932160, offset 0 00000400:00000200:0.0:1788205134.057405:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205134.057415:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.057421:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205134.057428:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205134.057433:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205134.057437:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205134.057439:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205134.057445:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59138:59138:256:4294967295] 192.168.201.151@tcp LPNI seq info [59118:59118:8:4294967295] 00000400:00000200:0.0:1788205134.057449:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.057460:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205134.057467:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.057474:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205134.057482:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205134.057486:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205134.057492:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205134.057515:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205134.057521:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.057524:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205134.057534:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205134.057539:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205134.057546:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205134.057549:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205134.057554:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205134.057558:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057569:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.057573:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057580:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.057582:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205134.057590:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205134.057593:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205134.057597:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301265410 00000100:00000010:0.0:1788205134.057602:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205134.057609:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205134.057616:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301265410 00000100:00000010:0.0:1788205134.057619:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9c60. 00000100:00000001:0.0:1788205134.057622:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205134.057626:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205134.057629:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.057632:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205134.057637:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205134.057647:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205134.057649:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205134.057651:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205134.057653:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205134.057655:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205134.057658:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205134.057661:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205134.057670:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057677:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205134.057679:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057682:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.057684:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205134.057688:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057695:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205134.057699:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057706:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205134.057709:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057711:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.057712:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057714:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205134.057716:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205134.057719:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205134.057724:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205134.057727:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205134.057731:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205134.057739:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205932160:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205134.057744:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205134.057746:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057752:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205134.057754:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.057756:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205134.057758:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205134.057759:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205134.057762:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932288:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205134.057765:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205134.057766:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205134.057768:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.057769:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205134.057772:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205134.057774:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205134.057777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205134.057780:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205134.057783:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205134.057786:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932288, portal 10 00000100:00000001:0.0:1788205134.057788:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205134.057789:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834048 : -113511408717568 : ffff98c310ce3100) 00000100:00000040:0.0:1788205134.057795:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057802:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205134.057804:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205134.057806:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205134.057808:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205932288, offset 0 00000400:00000200:0.0:1788205134.057812:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205134.057816:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.057819:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205134.057822:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205134.057824:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205134.057826:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205134.057829:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205134.057832:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59139:59139:256:4294967295] 192.168.201.151@tcp LPNI seq info [59119:59119:8:4294967295] 00000400:00000200:0.0:1788205134.057836:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.057842:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205134.057847:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.057850:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205134.057853:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205134.057856:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205134.057861:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205134.057863:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205134.057867:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.057869:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205134.057872:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205134.057876:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205134.057878:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205134.057880:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205134.057883:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205134.057886:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057894:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.057897:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057903:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.057904:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205134.057907:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205134.057912:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205134.057914:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205134.057916:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301261467 00000100:00000001:0.0:1788205134.057919:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.057920:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301261467 00000100:00000001:0.0:1788205134.057922:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.057924:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205134.057926:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.057927:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205134.057931:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205134.057937:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205134.057938:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205134.057940:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205134.057942:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205134.057943:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205134.057945:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205134.057948:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205134.057955:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057960:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205134.057961:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057962:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.057963:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205134.057966:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057972:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205134.057975:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.057981:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205134.057984:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057985:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.057986:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.057988:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205134.057991:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205134.057993:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205134.057996:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205134.057998:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205134.058001:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205134.058009:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932288:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205134.058013:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205134.058016:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058022:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205134.058024:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.058026:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205134.058027:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205134.058029:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205134.058032:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932416:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205134.058035:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205134.058037:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205134.058039:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.058040:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058044:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205134.058046:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205134.058048:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205134.058051:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205134.058054:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205134.058056:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932416, portal 4 00000100:00000001:0.0:1788205134.058058:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205134.058059:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300833152 : -113511408718464 : ffff98c310ce2d80) 00000100:00000040:0.0:1788205134.058065:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058071:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205134.058073:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205134.058076:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205134.058078:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205932416, offset 0 00000400:00000200:0.0:1788205134.058081:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205134.058084:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.058088:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205134.058091:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205134.058093:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205134.058095:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205134.058097:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205134.058100:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59140:59140:256:4294967295] 192.168.201.151@tcp LPNI seq info [59120:59120:8:4294967295] 00000400:00000200:0.0:1788205134.058104:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.058109:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205134.058113:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.058116:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205134.058120:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205134.058122:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205134.058126:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205134.058129:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205134.058132:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.058134:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205134.058137:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205134.058140:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205134.058143:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205134.058145:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205134.058148:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205134.058151:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058159:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058161:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058167:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.058168:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205134.058172:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205134.058175:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205134.058178:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205134.058180:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301261353 00000100:00000001:0.0:1788205134.058182:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.058184:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301261353 00000100:00000001:0.0:1788205134.058186:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.058188:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205134.058190:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.058192:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205134.058196:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205134.058203:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205134.058205:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205134.058206:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205134.058221:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205134.058223:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205134.058225:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205134.058228:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205134.058235:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058246:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205134.058247:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058249:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.058250:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205134.058253:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058258:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205134.058261:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058267:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205134.058269:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058270:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.058272:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058274:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205134.058276:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205134.058279:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205134.058281:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205134.058283:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205134.058287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205134.058294:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932416:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205134.058298:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205134.058302:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058307:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205134.058309:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.058311:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205134.058313:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205134.058315:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205134.058318:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932544:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205134.058321:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205134.058323:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205134.058324:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205134.058326:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058329:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205134.058331:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205134.058334:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205134.058336:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205134.058340:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205134.058342:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932544, portal 4 00000100:00000001:0.0:1788205134.058344:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205134.058345:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300834944 : -113511408716672 : ffff98c310ce3480) 00000100:00000040:0.0:1788205134.058350:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058356:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205134.058359:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205134.058361:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205134.058391:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205932544, offset 0 00000400:00000200:0.0:1788205134.058396:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205134.058400:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.058403:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205134.058406:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205134.058408:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205134.058411:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205134.058413:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205134.058418:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59141:59141:256:4294967295] 192.168.201.151@tcp LPNI seq info [59121:59121:8:4294967295] 00000400:00000200:0.0:1788205134.058422:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205134.058428:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205134.058433:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.058436:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205134.058439:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205134.058442:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788205134.058446:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205134.058448:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788205134.058452:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205134.058454:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205134.058457:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205134.058460:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205134.058463:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205134.058465:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205134.058467:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205134.058470:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058477:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058479:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058485:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.058487:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205134.058490:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205134.058493:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205134.058496:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205134.058498:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301261360 00000100:00000001:0.0:1788205134.058501:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.058503:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301261360 00000100:00000001:0.0:1788205134.058505:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301265410 00000100:00000001:0.0:1788205134.058507:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205134.058509:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.058511:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205134.058515:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205134.058522:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205134.058523:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205134.058525:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205134.058528:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205134.058529:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205134.058531:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205134.058534:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205134.058542:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058549:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205134.058550:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058552:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.058553:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205134.058557:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058563:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205134.058567:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058573:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205134.058575:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058576:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205134.058577:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058578:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205134.058580:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205134.058583:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205134.058586:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205134.058588:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205134.058592:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205134.058598:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932544:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205134.058602:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058604:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058607:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0380 x1875062205932160/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058613:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205134.058616:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058619:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205134.058622:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205134.058624:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205134.058626:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205134.058628:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205134.058631:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21400. 02000000:00000001:0.0:1788205134.058634:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205134.058636:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205134.058637:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0380. 00000100:00000001:0.0:1788205134.058641:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205134.058642:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058643:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058646:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3100 x1875062205932288/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058651:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205134.058652:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058653:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205134.058656:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205134.058657:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205134.058659:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205134.058661:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205134.058662:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c20400. 02000000:00000001:0.0:1788205134.058664:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205134.058665:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205134.058667:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3100. 00000100:00000001:0.0:1788205134.058669:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205134.058670:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058671:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058673:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2d80 x1875062205932416/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058679:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205134.058680:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058681:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205134.058682:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205134.058684:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205134.058685:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205134.058686:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205134.058688:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:0.0:1788205134.058689:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205134.058690:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205134.058691:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2d80. 00000100:00000001:0.0:1788205134.058693:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205134.058694:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058696:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205134.058698:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce3480 x1875062205932544/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205134.058704:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205134.058705:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205134.058706:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205134.058708:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205134.058709:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205134.058710:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205134.058711:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205134.058713:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:0.0:1788205134.058714:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205134.058716:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205134.058717:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce3480. 00000100:00000001:0.0:1788205134.058718:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205134.058720:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058722:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205134.058730:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205134.058732:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205134.058734:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205134.058735:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205134.248589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205134.248599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205134.248606:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205134.248607:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205134.248610:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205134.248611:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205134.248613:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205134.248614:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205134.248680:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205134.248685:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205134.248690:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205134.568546:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205134.568557:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205134.568560:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.568568:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205134.568580:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205134.568584:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205134.568588:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205134.568591:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205134.568594:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.016515:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205135.016523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205135.016526:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.016532:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205135.016537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205135.016541:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.016544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.016548:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.016551:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205135.080484:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205135.080499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205135.080501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.080505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788205135.080507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205135.080511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205135.080512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205135.080515:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205135.080517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205135.080520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205135.080521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205135.080525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205135.080526:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205135.080530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.080530:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205135.080536:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205135.080538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205135.080541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205135.272627:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205135.272637:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205135.272645:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205135.272646:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205135.272650:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205135.272651:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205135.272655:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205135.272656:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205135.272876:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205135.272880:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205135.272884:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205135.592516:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205135.592524:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205135.592526:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.592532:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205135.592535:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205135.592539:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.592541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.592544:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.592547:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205136.040496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205136.040502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205136.040504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205136.040509:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205136.040513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205136.040516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.040519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205136.040522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.040525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205136.104518:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205136.104518:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205136.104528:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205136.104531:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205136.104537:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205136.104540:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205136.104543:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205136.104544:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205136.104550:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205136.104550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205136.104555:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205136.104556:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205136.104560:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205136.104562:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205136.104568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205136.104572:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205136.104575:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205136.104578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205136.296950:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205136.296961:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205136.296968:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205136.296970:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205136.296973:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205136.296975:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205136.296978:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205136.296980:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205136.297223:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205136.297228:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205136.297232:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205136.616487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205136.616494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205136.616496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205136.616501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205136.616504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205136.616508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.616511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205136.616514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.616517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205137.066540:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205137.066548:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205137.066551:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205137.066557:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205137.066561:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205137.066565:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.066568:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205137.066571:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.066574:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205137.128502:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205137.128510:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205137.128512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205137.128518:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205137.128521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205137.128524:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205137.128527:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205137.128530:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205137.128533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205137.129078:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205137.129082:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205137.129083:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205137.129086:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205137.129088:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205137.129090:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.129092:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205137.129094:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.129096:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205137.320699:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205137.320706:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205137.320712:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205137.320713:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205137.320716:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205137.320717:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205137.320720:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205137.320721:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205137.320801:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205137.320808:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205137.320813:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205137.640495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205137.640503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205137.640506:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205137.640512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205137.640515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205137.640519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.640522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205137.640524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.640528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205138.089035:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205138.089042:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205138.089044:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205138.089049:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205138.089052:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205138.089056:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.089058:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205138.089061:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.089069:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205138.153248:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205138.153254:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205138.153256:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205138.153260:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205138.153262:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205138.153265:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.153267:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205138.153269:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.153271:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.153404:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205138.153407:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205138.153409:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205138.153411:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205138.153414:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205138.153416:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205138.153418:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205138.153419:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205138.153421:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205138.344486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205138.344493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205138.344499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205138.344500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205138.344503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205138.344504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205138.344507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205138.344508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205138.344641:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205138.344645:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205138.344648:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205138.664490:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301265410, at 4301265920 00000100:00000001:3.0:1788205138.664498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205138.664504:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205138.664506:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205138.664507:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00100000:3.0:1788205138.664511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205138.664515:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205138.664518:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.664521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205138.664524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.664527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205138.667234:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301265410, at 4301265922 00000100:00000010:1.0:1788205138.667247:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9c60. 00000100:00000001:3.0:1788205139.114026:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205139.114031:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205139.114033:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205139.114037:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205139.114040:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205139.114043:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.114046:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205139.114048:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.114051:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205139.176691:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788205139.176692:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205139.176698:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788205139.176701:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205139.176702:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.176706:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205139.176708:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:1.0:1788205139.176711:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205139.176714:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205139.176716:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205139.176719:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.176724:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788205139.176725:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6594 00000100:00000001:3.0:1788205139.176727:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.176732:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205139.176737:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205139.176743:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205139.176746:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205139.176748:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.176752:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.176755:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.176757:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205139.176761:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205139.176763:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205139.176765:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205139.176768:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205139.176771:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205139.176773:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205139.176776:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.176778:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.176780:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.176782:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205139.176785:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205139.176788:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205139.176791:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205139.176793:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205139.176797:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205139.176799:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.176801:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205139.176804:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205139.177092:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788205139.177097:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205139.177102:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205139.177103:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177106:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205139.177108:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205139.177112:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205139.177129:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788205139.177136:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205139.177146:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205139.177156:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205139.177160:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.177165:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205139.177191:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205139.177192:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205139.177195:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788205139.177198:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788205139.177198:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205139.177202:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205139.177205:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205139.177206:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205139.177207:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205139.177210:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205139.177213:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205139.177216:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788205139.177220:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205139.177222:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205139.177224:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205139.177230:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6594 00000400:00000200:1.0:1788205139.177236:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205139.177240:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205139.177243:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205139.177246:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177249:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177251:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205139.177253:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205139.177254:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205139.177257:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788205139.177259:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205139.177260:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788205139.177263:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205139.177267:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:0.0:1788205139.177268:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205139.177270:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205139.177275:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205139.177281:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205139.177283:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205139.177287:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.177287:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205139.177292:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205139.177292:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177296:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205139.177297:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205139.177299:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:0.0:1788205139.177302:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205139.177304:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00100000:0.0:1788205139.177309:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205932672:192.168.201.151@tcp:250:kworker.0 00000100:00080000:1.0:1788205139.177311:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205139.177318:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205139.177319:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205139.177322:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205139.177324:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205139.177328:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205139.177328:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205139.177332:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205139.177334:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177337:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:0.0:1788205139.177339:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 00000008:00100000:1.0:1788205139.177342:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788205139.177343:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788205139.177348:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177351:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205139.177355:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:1.0:1788205139.177358:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000400:00000010:0.0:1788205139.177361:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000020:00000040:1.0:1788205139.177362:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:0.0:1788205139.177395:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205139.177401:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932672, portal 25 00000100:00000001:0.0:1788205139.177404:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205139.177406:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000040:0.0:1788205139.177412:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177419:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205139.177422:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205139.177424:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205139.177427:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205932672, offset 0 00000400:00000200:0.0:1788205139.177432:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205139.177439:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.177444:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205139.177449:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205139.177453:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205139.177456:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205139.177459:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205139.177462:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59142:59142:256:4294967295] 192.168.201.151@tcp LPNI seq info [59122:59122:8:4294967295] 00000400:00000200:0.0:1788205139.177466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.177474:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205139.177480:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.177484:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205139.177491:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205139.177494:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788205139.177499:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205139.177502:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788205139.177506:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.177508:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205139.177514:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205139.177517:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205139.177522:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205139.177523:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205139.177526:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205139.177529:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177535:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.177538:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177543:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.177544:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205139.177548:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205139.177550:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205139.177552:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301270530 00000100:00000010:0.0:1788205139.177556:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9a20. 00000100:00000001:0.0:1788205139.177560:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205139.177564:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301270530 00000100:00000010:0.0:1788205139.177566:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9b40. 00000100:00000001:0.0:1788205139.177568:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205139.177570:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205139.177572:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.177574:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205139.177576:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205139.177581:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205139.177582:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205139.177583:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205139.177585:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205139.177586:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205139.177588:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205139.177590:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205139.177596:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177602:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205139.177604:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.177606:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.177608:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205139.177611:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177618:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205139.177622:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177628:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205139.177630:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.177632:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.177633:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.177635:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205139.177637:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205139.177639:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205139.177642:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205139.177645:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205139.177648:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205139.177657:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205932672:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205139.177661:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.177664:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.177666:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205932672/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.177672:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205139.177675:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205139.177677:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205139.177683:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1788205139.177683:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1788205139.177685:0:2345:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:1.0:1788205139.177686:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205139.177688:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177690:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1788205139.177694:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1788205139.177694:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1788205139.177699:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc400. 02000000:00000001:1.0:1788205139.177699:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.177704:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:1.0:1788205139.177707:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:0.0:1788205139.177708:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205139.177711:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:1.0:1788205139.177713:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.177716:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205139.177718:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.177721:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205139.177722:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205139.177728:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205139.177730:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.177733:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205139.177735:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205139.177736:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177739:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.177745:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205139.177764:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205139.177767:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205139.177770:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205139.177771:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205139.177773:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205139.177775:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205139.177778:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6594 00000400:00000200:1.0:1788205139.177782:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205139.177785:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205139.177787:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205139.177790:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177792:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177794:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177796:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205139.177798:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205139.177799:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205139.177800:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205139.177802:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205139.177804:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205139.177806:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205139.177819:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177820:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177822:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177824:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205139.177826:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205139.177828:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177830:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205139.177833:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205139.177844:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205139.177846:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177848:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205139.177851:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205139.177853:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177854:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205139.177858:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000020:00000040:1.0:1788205139.177860:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205139.177862:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205139.177863:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177865:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205139.177866:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205139.177868:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205139.177871:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788205139.177874:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205139.177879:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205139.177885:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205139.177887:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.177890:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205139.177896:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205139.177898:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205139.177902:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205139.177903:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205139.177905:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205139.177907:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205139.177909:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6594 00000400:00000200:1.0:1788205139.177913:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205139.177916:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205139.177918:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205139.177921:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177923:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177924:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177927:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205139.177929:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205139.177930:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205139.177931:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205139.177933:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205139.177935:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205139.177936:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205139.177939:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177941:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205139.177942:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177944:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205139.177946:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205139.177949:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205139.177951:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205139.177952:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205139.177955:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205139.177957:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177958:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205139.177960:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205139.177962:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177964:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205139.177967:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788205139.177969:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205139.177971:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205139.177973:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205139.177974:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205139.177975:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205139.177977:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205139.177981:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000100:00000001:1.0:1788205139.177984:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205139.177989:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205139.177996:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205139.177998:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.178001:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205139.178008:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205139.178012:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6604) 00000100:00000001:0.0:1788205139.178183:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205139.178185:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205139.178186:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205139.178189:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178194:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205139.178195:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178196:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205139.178197:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205139.178199:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205139.178200:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932800:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205139.178202:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205139.178203:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205139.178205:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178206:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205139.178208:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205139.178210:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205139.178212:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205139.178214:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205139.178216:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205139.178218:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932800, portal 10 00000100:00000001:0.0:1788205139.178219:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205139.178220:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:0.0:1788205139.178225:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178229:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205139.178230:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205139.178232:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205139.178233:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205932800, offset 0 00000400:00000200:0.0:1788205139.178235:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205139.178238:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.178240:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205139.178242:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205139.178244:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205139.178246:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205139.178247:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205139.178249:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59143:59143:256:4294967295] 192.168.201.151@tcp LPNI seq info [59123:59123:8:4294967295] 00000400:00000200:0.0:1788205139.178252:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.178256:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205139.178258:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.178260:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205139.178262:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205139.178264:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788205139.178267:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205139.178268:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788205139.178270:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.178272:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205139.178274:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205139.178276:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205139.178278:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205139.178279:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205139.178281:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205139.178283:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178288:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.178290:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178294:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.178294:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205139.178297:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205139.178299:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205139.178301:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205139.178302:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301266587 00000100:00000001:0.0:1788205139.178304:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.178305:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301266587 00000100:00000001:0.0:1788205139.178307:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.178308:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205139.178309:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.178310:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205139.178312:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205139.178316:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205139.178317:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205139.178318:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205139.178320:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205139.178320:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205139.178322:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205139.178324:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205139.178328:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178332:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205139.178333:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178334:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.178334:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205139.178336:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178340:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205139.178342:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178347:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205139.178349:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178351:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.178352:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178354:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205139.178356:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205139.178359:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205139.178377:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205139.178380:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205139.178384:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205139.178392:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932800:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205139.178395:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205139.178398:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178405:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205139.178406:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178408:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205139.178410:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205139.178412:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205139.178414:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932928:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205139.178418:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205139.178420:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205139.178421:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178422:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205139.178424:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205139.178425:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205139.178428:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205139.178430:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205139.178432:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205139.178434:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205932928, portal 4 00000100:00000001:0.0:1788205139.178435:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205139.178436:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:0.0:1788205139.178440:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178443:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205139.178445:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205139.178446:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205139.178447:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205932928, offset 0 00000400:00000200:0.0:1788205139.178449:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205139.178451:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.178453:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205139.178455:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205139.178457:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205139.178458:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205139.178460:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205139.178462:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59144:59144:256:4294967295] 192.168.201.151@tcp LPNI seq info [59124:59124:8:4294967295] 00000400:00000200:0.0:1788205139.178464:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.178467:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205139.178470:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.178472:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205139.178474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205139.178476:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788205139.178478:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205139.178480:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788205139.178482:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.178483:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205139.178485:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205139.178487:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205139.178489:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205139.178490:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205139.178492:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205139.178494:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178498:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.178500:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178503:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.178504:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205139.178506:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205139.178508:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205139.178509:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205139.178510:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301266474 00000100:00000001:0.0:1788205139.178512:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.178513:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301266474 00000100:00000001:0.0:1788205139.178514:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.178515:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205139.178517:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.178518:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205139.178520:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205139.178524:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205139.178524:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205139.178525:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205139.178526:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205139.178527:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205139.178528:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205139.178530:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205139.178534:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178541:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205139.178542:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178543:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.178544:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205139.178546:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178549:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205139.178551:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178555:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205139.178556:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178557:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.178557:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.178558:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205139.178560:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205139.178561:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205139.178563:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205139.178564:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205139.178566:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205139.178570:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205932928:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205139.178571:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205139.178573:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.178577:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205139.178578:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178579:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205139.178580:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205139.178582:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205139.178583:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933056:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205139.178585:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205139.178586:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205139.178587:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205139.178587:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205139.179230:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205139.179234:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205139.179238:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205139.179240:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205139.179244:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205139.179247:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933056, portal 4 00000100:00000001:0.0:1788205139.179249:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205139.179251:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:0.0:1788205139.179258:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179264:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205139.179266:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205139.179268:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205139.179271:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205933056, offset 0 00000400:00000200:0.0:1788205139.179275:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205139.179279:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.179283:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205139.179286:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205139.179288:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205139.179291:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205139.179293:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205139.179296:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59145:59145:256:4294967295] 192.168.201.151@tcp LPNI seq info [59125:59125:8:4294967295] 00000400:00000200:0.0:1788205139.179299:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205139.179305:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205139.179310:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.179313:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205139.179317:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205139.179320:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788205139.179324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205139.179327:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788205139.179330:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205139.179332:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205139.179336:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205139.179339:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205139.179342:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205139.179344:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205139.179346:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205139.179349:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179356:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.179360:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179384:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.179386:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205139.179389:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205139.179392:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205139.179395:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205139.179397:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301266481 00000100:00000001:0.0:1788205139.179400:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.179402:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301266481 00000100:00000001:0.0:1788205139.179403:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301270530 00000100:00000001:0.0:1788205139.179405:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205139.179407:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.179409:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205139.179413:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205139.179421:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205139.179422:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205139.179424:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205139.179427:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205139.179428:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205139.179430:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205139.179433:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205139.179442:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179449:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205139.179450:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179452:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.179454:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205139.179457:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179463:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205139.179466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179473:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205139.179475:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179476:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205139.179478:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179479:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205139.179481:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205139.179483:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205139.179486:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205139.179488:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205139.179492:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205139.179499:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933056:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205139.179502:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179504:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.179507:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205932800/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179513:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205139.179515:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205139.179517:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205139.179520:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205139.179521:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205139.179523:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205139.179525:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205139.179527:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:0.0:1788205139.179530:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205139.179531:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205139.179533:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:0.0:1788205139.179535:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205139.179536:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179537:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.179540:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205932928/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179546:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205139.179547:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205139.179548:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8000. 02000000:00000001:0.0:1788205139.179551:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205139.179552:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205139.179553:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205139.179554:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205139.179555:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:0.0:1788205139.179558:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205139.179559:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205139.179560:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:0.0:1788205139.179562:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205139.179564:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179565:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205139.179568:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205933056/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205139.179574:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205139.179575:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205139.179577:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205139.179579:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205139.179580:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205139.179581:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205139.179582:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205139.179584:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:0.0:1788205139.179586:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205139.179587:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205139.179588:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:0.0:1788205139.179590:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205139.179592:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179594:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205139.179600:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205139.179601:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205139.179603:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205139.179605:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205139.368591:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205139.368599:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205139.368605:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205139.368606:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205139.368609:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205139.368611:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205139.368614:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205139.368615:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205139.368727:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205139.368730:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205139.368733:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205139.688514:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205139.688522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205139.688525:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205139.688530:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205139.688535:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205139.688539:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.688542:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205139.688544:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.688547:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.136513:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205140.136523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205140.136526:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.136533:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205140.136537:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205140.136543:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.136546:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.136549:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.136552:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.200521:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205140.200531:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205140.200534:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.200541:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205140.200546:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205140.200551:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.200553:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.200557:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.200561:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.200832:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205140.200839:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205140.200841:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205140.200847:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205140.200851:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205140.200854:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.200857:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205140.200859:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.200862:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205140.392517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205140.392527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205140.392535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205140.392536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205140.392540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205140.392542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205140.392546:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205140.392548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205140.392622:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205140.392627:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205140.392631:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205140.712537:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205140.712548:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205140.712550:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.712559:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205140.712563:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205140.712568:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.712571:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.712574:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.712578:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205141.161234:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205141.161241:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205141.161243:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205141.161248:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205141.161252:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205141.161256:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.161258:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205141.161261:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.161264:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205141.224430:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205141.224437:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205141.224439:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205141.224455:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205141.224459:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205141.224466:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.224469:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205141.224471:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.224474:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205141.224777:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205141.224784:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205141.224785:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205141.224790:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205141.224793:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205141.224796:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.224798:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205141.224800:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.224803:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417171:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205141.417178:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417183:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205141.417186:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205141.417188:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205141.417191:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417194:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205141.417196:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417198:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417200:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417204:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205141.417206:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205141.417208:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205141.417210:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205141.419240:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205141.419245:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205141.419248:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205141.736512:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205141.736523:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205141.736526:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205141.736532:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205141.736538:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205141.736544:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.736547:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205141.736551:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.736554:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.184448:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205142.184486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205142.184489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205142.184494:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205142.184497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205142.184501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.184503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205142.184506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.184513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.248511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205142.248519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205142.248520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205142.248525:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788205142.248527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205142.248531:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205142.248532:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205142.248535:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.248537:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205142.248540:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205142.248542:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205142.248546:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205142.248547:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205142.248551:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.248552:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205142.248556:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205142.248560:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205142.248563:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205142.440516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205142.440522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205142.440527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205142.440528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205142.440532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205142.440533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205142.440536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205142.440537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205142.440632:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205142.440635:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205142.440637:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205142.760500:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205142.760510:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205142.760513:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205142.760520:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205142.760524:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205142.760529:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.760532:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205142.760535:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.760538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205143.208495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205143.208503:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205143.208505:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205143.208510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205143.208513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205143.208517:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.208519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205143.208522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.208525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205143.272491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205143.272498:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205143.272500:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205143.272505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205143.272508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205143.272512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.272514:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205143.272516:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.272520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205143.273198:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205143.273201:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205143.273202:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205143.273205:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205143.273212:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205143.273214:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205143.273216:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205143.273217:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205143.273219:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464611:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205143.464619:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205143.464621:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205143.464625:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464630:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205143.464632:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464634:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464635:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464640:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205143.464641:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464644:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205143.464646:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205143.464647:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205143.464648:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464650:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205143.464651:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464652:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464654:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464657:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205143.464659:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205143.464660:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205143.464661:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464662:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205143.464664:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464665:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205143.464667:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205143.466907:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205143.466912:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205143.466916:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205143.784498:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301270530, at 4301271040 00000100:00000010:0.0:1788205143.784511:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9a20. 00000100:00000040:1.0:1788205143.785122:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301270530, at 4301271040 00000100:00000010:1.0:1788205143.785129:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9b40. 00000100:00000001:3.0:1788205143.785805:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205143.785809:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205143.785811:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205143.785815:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205143.785818:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205143.785821:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.785823:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205143.785827:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.785830:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.232670:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205144.232682:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.232684:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.232691:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205144.232696:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205144.232711:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.232714:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205144.232718:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.232722:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.296519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205144.296531:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.296534:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.296542:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788205144.296545:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205144.296549:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205144.296551:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205144.296553:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205144.296556:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205144.296556:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788205144.296562:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205144.296563:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.296565:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205144.296567:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205144.296570:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205144.296573:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.296577:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205144.296578:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6599 00000100:00100000:1.0:1788205144.296585:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000400:00000200:0.0:1788205144.296591:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205144.296597:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205144.296600:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205144.296607:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205144.296607:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205144.296613:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205144.296616:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205144.296617:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296624:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788205144.296626:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788205144.296628:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205144.296631:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000100:00000001:1.0:1788205144.296631:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205144.296636:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788205144.296640:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205144.296641:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205144.296647:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205144.296650:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205144.296654:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296656:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.296658:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296659:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205144.296661:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205144.296664:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296665:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205144.296667:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205144.296670:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205144.296672:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296675:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205144.296678:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205144.296688:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205144.296691:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205144.296695:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205144.296696:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296699:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205144.296701:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205144.296705:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205144.296714:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788205144.296720:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205144.296729:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205144.296737:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205144.296741:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205144.296745:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205144.296764:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205144.296767:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205144.296770:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205144.296771:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205144.296772:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205144.296774:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205144.296775:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6599 00000400:00000200:0.0:1788205144.296777:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205144.296779:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205144.296781:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205144.296783:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296785:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.296786:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296787:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205144.296788:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205144.296789:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205144.296791:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205144.296792:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205144.296794:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205144.296795:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205144.296797:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296798:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.296800:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296801:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205144.296802:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205144.296804:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296805:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205144.296806:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205144.296809:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788205144.296811:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205144.296814:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296815:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205144.296816:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1788205144.296819:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205144.296820:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1788205144.296823:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296826:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205144.296830:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000100:00000040:3.0:1788205144.296831:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205144.296832:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205144.296836:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205144.296838:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296841:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205144.296842:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205144.296845:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205144.296849:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:3.0:1788205144.296852:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205144.296854:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.296856:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205144.296862:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205144.296863:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205144.296867:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205144.296873:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296875:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205144.296881:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.296881:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205933184:192.168.201.151@tcp:250:kworker.0 00000100:00000040:0.0:1788205144.296889:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.296890:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205144.296897:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205144.296901:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296903:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.296906:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788205144.296907:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205144.296911:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205144.296912:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205144.296914:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:3.0:1788205144.296915:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 00000100:00000001:0.0:1788205144.296917:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205144.296921:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6599 02000000:00000001:3.0:1788205144.296921:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205144.296927:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205144.296929:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205144.296931:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205144.296932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788205144.296935:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296938:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.296941:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:3.0:1788205144.296941:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788205144.296943:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205144.296947:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205144.296948:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205144.296949:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205144.296950:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205144.296951:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205144.296952:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000010:3.0:1788205144.296953:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000040:0.0:1788205144.296955:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296958:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.296959:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296961:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205144.296962:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000200:3.0:1788205144.296964:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933184, portal 25 00000100:00080000:0.0:1788205144.296965:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205144.296968:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.296970:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788205144.296971:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205144.296974:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205144.296976:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 02000000:00000001:0.0:1788205144.296977:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296981:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205144.296984:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205144.296986:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.296988:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205144.296989:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788205144.296991:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205144.296995:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205144.296999:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205144.297001:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.297004:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205144.297006:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205144.297007:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205144.297011:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000100:00000001:0.0:1788205144.297014:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297019:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788205144.297020:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205144.297029:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205144.297044:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205144.297048:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205144.297056:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205144.297059:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205144.297063:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205144.297064:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205144.297065:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205144.297067:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205144.297069:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6599 00000400:00000200:0.0:1788205144.297072:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205144.297074:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205144.297075:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205144.297078:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.297079:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.297081:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.297082:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205144.297084:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205144.297084:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205144.297085:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205144.297087:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205144.297088:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205144.297089:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205144.297091:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.297092:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205144.297093:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205144.297094:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205144.297096:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205144.297097:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205144.297098:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205144.297099:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205144.297101:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205144.297102:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.297104:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205144.297105:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205144.297106:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.297107:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205144.297110:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205144.297111:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205144.297113:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205144.297114:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205144.297115:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205144.297116:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205144.297117:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205144.297120:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788205144.297123:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205144.297125:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205144.297130:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205144.297131:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205144.297133:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205144.297138:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205144.297143:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6609) 00000100:00000040:3.0:1788205144.297262:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205144.297267:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205144.297272:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205933184, offset 0 00000400:00000200:3.0:1788205144.297294:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205144.297305:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.297312:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205144.297319:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205144.297325:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205144.297330:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205144.297333:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205144.297338:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59146:59146:256:4294967295] 192.168.201.151@tcp LPNI seq info [59126:59126:8:4294967295] 00000400:00000200:3.0:1788205144.297343:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.297355:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205144.297383:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.297391:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205144.297399:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205144.297404:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205144.297411:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205144.297434:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205144.297439:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.297443:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205144.297451:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205144.297456:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205144.297463:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205144.297466:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205144.297471:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205144.297477:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297488:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.297493:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297501:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297503:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205144.297511:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205144.297516:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205144.297520:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301275650 00000100:00000010:3.0:1788205144.297525:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788205144.297534:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205144.297541:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301275650 00000100:00000010:3.0:1788205144.297545:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4a20. 00000100:00000001:3.0:1788205144.297548:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205144.297552:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205144.297555:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297558:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205144.297563:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205144/real 1788205144] req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205144.297573:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205144.297575:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205144.297578:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205144.297581:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205144.297583:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205144.297585:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205144.297589:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205144.297598:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297606:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205144.297608:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297610:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.297613:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205144.297618:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297625:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205144.297630:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297637:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205144.297640:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297642:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.297643:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297647:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205144.297650:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205144.297653:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205144.297658:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205144.297661:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205144.297665:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205144.297673:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205933184:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205144.297678:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297680:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.297683:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205933184/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297690:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205144.297693:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205144.297695:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205144.297698:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205144.297700:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205144.297702:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205144.297704:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205144.297707:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:3.0:1788205144.297711:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205144.297713:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205144.297715:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205144.297718:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205144.297719:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297722:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.297731:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.297732:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297734:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205144.297735:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205144.297737:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205144.297740:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297751:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205144.297753:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.297755:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205144.297757:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205144.297759:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.297762:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933312:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205144.297766:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205144.297768:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205144.297769:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.297771:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205144.297774:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205144.297776:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205144.297781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205144.297784:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205144.297800:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205144.297803:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933312, portal 10 00000100:00000001:3.0:1788205144.297805:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205144.297806:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205144.297813:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297820:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205144.297822:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205144.297824:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205144.297827:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205933312, offset 0 00000400:00000200:3.0:1788205144.297831:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205144.297835:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.297839:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205144.297842:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205144.297844:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205144.297847:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205144.297849:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205144.297853:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59147:59147:256:4294967295] 192.168.201.151@tcp LPNI seq info [59127:59127:8:4294967295] 00000400:00000200:3.0:1788205144.297858:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.297864:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205144.297869:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.297873:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205144.297877:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205144.297880:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205144.297884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205144.297886:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205144.297890:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.297892:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205144.297895:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205144.297898:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205144.297902:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205144.297904:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205144.297906:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205144.297910:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297918:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.297921:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297928:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297930:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205144.297933:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205144.297938:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205144.297941:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205144.297943:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301271707 00000100:00000001:3.0:1788205144.297946:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.297949:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301271707 00000100:00000001:3.0:1788205144.297951:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.297953:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205144.297956:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.297958:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205144.297962:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205144/real 1788205144] req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205144.297971:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205144.297972:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205144.297974:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205144.297977:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205144.297978:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205144.297981:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205144.297984:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205144.297993:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.297999:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205144.298000:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298002:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298013:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205144.298017:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298097:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205144.298103:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298111:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205144.298122:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298124:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298126:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298128:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205144.298130:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205144.298133:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205144.298138:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205144.298140:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205144.298144:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205144.298153:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933312:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205144.298158:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205144.298161:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298167:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205144.298169:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.298171:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205144.298173:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205144.298175:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.298178:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933440:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205144.298181:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205144.298183:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205144.298185:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.298187:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205144.298190:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205144.298192:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205144.298196:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205144.298199:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205144.298203:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205144.298206:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933440, portal 4 00000100:00000001:3.0:1788205144.298208:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205144.298209:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205144.298216:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298223:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205144.298226:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205144.298228:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205144.298230:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205933440, offset 0 00000400:00000200:3.0:1788205144.298235:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205144.298239:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.298242:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205144.298245:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205144.298248:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205144.298250:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205144.298253:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205144.298255:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59148:59148:256:4294967295] 192.168.201.151@tcp LPNI seq info [59128:59128:8:4294967295] 00000400:00000200:3.0:1788205144.298259:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.298266:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205144.298271:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.298274:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205144.298278:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205144.298281:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205144.298285:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205144.298287:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205144.298291:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.298293:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205144.298296:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205144.298299:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205144.298302:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205144.298304:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205144.298307:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205144.298311:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298319:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.298322:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298329:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.298330:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205144.298334:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205144.298338:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205144.298342:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205144.298344:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301271593 00000100:00000001:3.0:1788205144.298346:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.298349:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301271593 00000100:00000001:3.0:1788205144.298351:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.298353:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205144.298355:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.298357:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205144.298361:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205144/real 1788205144] req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205144.298391:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205144.298393:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205144.298395:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205144.298398:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205144.298399:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205144.298402:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205144.298405:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205144.298414:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298420:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205144.298422:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298424:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298425:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205144.298429:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298436:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205144.298440:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298447:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205144.298450:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298452:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298453:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298455:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205144.298457:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205144.298460:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205144.298464:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205144.298466:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205144.298470:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205144.298479:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933440:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205144.298483:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205144.298486:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298493:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205144.298494:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.298497:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205144.298498:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205144.298500:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.298503:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933568:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205144.298506:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205144.298508:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205144.298510:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205144.298511:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205144.298515:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205144.298517:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205144.298520:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205144.298524:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205144.298527:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205144.298530:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933568, portal 4 00000100:00000001:3.0:1788205144.298531:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205144.298533:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:3.0:1788205144.298539:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298545:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205144.298547:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205144.298550:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205144.298552:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205933568, offset 0 00000400:00000200:3.0:1788205144.298556:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205144.298560:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.298564:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205144.298567:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205144.298569:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205144.298572:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205144.298574:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205144.298577:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59149:59149:256:4294967295] 192.168.201.151@tcp LPNI seq info [59129:59129:8:4294967295] 00000400:00000200:3.0:1788205144.298581:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205144.298587:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205144.298592:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.298595:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205144.298599:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205144.298602:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205144.298606:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205144.298608:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205144.298612:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205144.298614:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205144.298617:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205144.298621:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205144.298624:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205144.298625:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205144.298628:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205144.298632:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298639:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.298642:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205169 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298649:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.298650:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205144.298654:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205144.298657:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205144.298660:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205144.298662:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301271600 00000100:00000001:3.0:1788205144.298665:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.298667:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301271600 00000100:00000001:3.0:1788205144.298669:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301275650 00000100:00000001:3.0:1788205144.298672:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205144.298674:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.298676:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205144.298679:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205144/real 1788205144] req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205144.298687:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205144.298688:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205144.298690:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205144.298692:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205144.298694:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205144.298696:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205144.298700:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205144.298709:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298721:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205144.298723:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298725:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298726:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205144.298730:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298736:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205144.298740:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298747:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205144.298749:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298750:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205144.298752:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298754:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205144.298756:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205144.298758:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205144.298761:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205144.298764:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205144.298768:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205144.298775:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933568:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205144.298780:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298782:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.298785:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205933312/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298791:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205144.298793:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205144.298795:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205144.298798:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205144.298799:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205144.298801:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205144.298803:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205144.298805:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:3.0:1788205144.298807:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205144.298809:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205144.298810:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205144.298813:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205144.298814:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298816:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.298819:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205933440/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298826:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205144.298827:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205144.298829:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205144.298831:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205144.298832:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205144.298833:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205144.298835:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205144.298836:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:3.0:1788205144.298838:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205144.298839:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205144.298841:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205144.298843:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205144.298845:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298846:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205144.298849:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205933568/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205169 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205144.298855:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205144.298856:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205144.298857:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205144.298859:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205144.298860:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205144.298862:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205144.298864:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205144.298865:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788205144.298868:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205144.298869:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205144.298870:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:3.0:1788205144.298872:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205144.298873:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298875:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205144.298879:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.298881:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.298883:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205144.298885:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205144.488501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205144.488508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205144.488515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205144.488517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205144.488520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205144.488522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205144.488527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205144.488529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205144.489171:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205144.489175:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205144.489179:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205144.811227:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205144.811233:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205144.811236:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205144.811242:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205144.811246:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205144.811250:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205144.811253:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205144.811256:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205144.811260:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.256508:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205145.256519:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205145.256522:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.256529:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205145.256533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205145.256539:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.256542:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.256545:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.256549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205145.320491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205145.320491:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205145.320501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205145.320501:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205145.320505:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.320505:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205145.320515:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205145.320515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205145.320521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205145.320522:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205145.320527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.320527:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.320532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.320535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.320538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205145.320540:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205145.320543:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205145.320546:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205145.512558:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205145.512567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205145.512574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205145.512575:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205145.512579:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205145.512580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205145.512584:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205145.512585:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205145.512687:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205145.512691:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205145.512696:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205145.832518:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205145.832527:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205145.832529:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205145.832536:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205145.832540:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205145.832545:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205145.832548:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205145.832552:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205145.832555:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205146.280494:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205146.280503:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205146.280506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205146.280513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205146.280516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205146.280522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205146.280525:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205146.280527:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205146.280532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205146.344527:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205146.344535:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205146.344537:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205146.344543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205146.344546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205146.344551:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205146.344554:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205146.344557:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205146.344560:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205146.344659:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205146.344662:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205146.344664:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205146.344666:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205146.344668:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205146.344670:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205146.344672:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205146.344673:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205146.344676:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205146.536512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205146.536521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205146.536529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205146.536530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205146.536533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205146.536535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205146.536538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205146.536540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205146.536615:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205146.536622:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205146.536626:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205146.856547:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205146.856556:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205146.856559:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205146.856564:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205146.856568:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205146.856573:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205146.856577:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205146.856581:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205146.856584:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205147.305122:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205147.305129:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205147.305131:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205147.305136:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205147.305140:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205147.305143:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205147.305145:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205147.305148:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205147.305151:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205147.368517:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205147.368528:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205147.368531:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205147.368538:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205147.368543:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205147.368549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205147.368552:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205147.368556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205147.368560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205147.368567:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205147.368575:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205147.368577:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205147.368582:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205147.368586:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205147.368589:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205147.368597:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205147.368599:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205147.368602:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205147.560568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205147.560577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205147.560584:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205147.560586:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205147.560589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205147.560591:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205147.560595:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205147.560596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205147.560661:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205147.560666:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205147.560670:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205147.880507:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205147.880522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205147.880523:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205147.880528:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205147.880530:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205147.880533:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205147.880535:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205147.880537:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205147.880539:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.328433:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205148.328441:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205148.328443:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205148.328448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205148.328452:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205148.328455:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.328458:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205148.328459:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.328463:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205148.392472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205148.392480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205148.392483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205148.392488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205148.392493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205148.392494:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205148.392498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205148.392504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205148.392505:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205148.392507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.392509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205148.392512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205148.392526:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205148.392531:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205148.392536:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.392540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205148.392543:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.392546:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205148.584503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205148.584510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205148.584515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205148.584516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205148.584518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205148.584519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205148.584522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205148.584523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205148.584599:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205148.584606:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205148.584611:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205148.904486:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301275650, at 4301276160 00000100:00000010:1.0:1788205148.904502:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4a20. 00000100:00000040:0.0:1788205148.904508:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301275650, at 4301276160 00000100:00000010:0.0:1788205148.904517:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4ae0. 00000100:00000001:3.0:1788205148.904534:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205148.904539:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205148.904541:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205148.904545:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205148.904547:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205148.904551:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.904554:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205148.904556:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205148.904559:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.352472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205149.352480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205149.352482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.352486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205149.352489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205149.352493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.352495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.352497:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.352500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.416476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205149.416476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205149.416483:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205149.416483:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205149.416485:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.416487:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205149.416491:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205149.416492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205149.416495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205149.416498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205149.416499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205149.416502:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205149.416504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205149.416505:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.416511:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.416515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.416517:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.416520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205149.416533:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205149.416537:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205149.416540:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205149.416543:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205149.416547:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205149.416554:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6604 00000400:00000200:1.0:1788205149.416563:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205149.416569:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205149.416573:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205149.416576:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416579:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.416581:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416583:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205149.416587:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205149.416588:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205149.416590:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205149.416592:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205149.416594:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205149.416595:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205149.416597:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416598:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.416599:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416600:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205149.416602:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205149.416604:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416605:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205149.416607:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205149.416610:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205149.416611:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.416613:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205149.416615:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205149.416630:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788205149.416635:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205149.416638:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205149.416639:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.416641:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205149.416643:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205149.416646:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205149.416652:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788205149.416656:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205149.416663:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205149.416670:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205149.416674:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205149.416678:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205149.416698:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205149.416701:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205149.416704:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205149.416705:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205149.416707:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205149.416710:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205149.416712:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6604 00000400:00000200:1.0:1788205149.416716:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205149.416719:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205149.416721:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205149.416725:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416727:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.416730:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416732:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205149.416734:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788205149.416735:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205149.416739:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:3.0:1788205149.416740:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788205149.416741:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205149.416744:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788205149.416745:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205149.416749:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205149.416754:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205149.416754:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205149.416759:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416765:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.416770:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205149.416771:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.416774:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.416775:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205149.416779:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1788205149.416781:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205149.416783:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788205149.416787:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416789:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205149.416791:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205149.416795:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205149.416797:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205149.416799:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.416803:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205149.416808:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205149.416808:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205933696:192.168.201.151@tcp:250:kworker.0 00000008:00000001:1.0:1788205149.416814:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.416816:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205149.416819:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.416822:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1788205149.416825:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 02000000:00000001:3.0:1788205149.416826:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205149.416831:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205149.416831:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205149.416836:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205149.416839:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205149.416839:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 00000100:00000001:1.0:1788205149.416842:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205149.416843:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205149.416845:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205149.416848:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205149.416852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:1.0:1788205149.416854:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bcc00. 00000400:00000010:3.0:1788205149.416857:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:1.0:1788205149.416860:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205149.416865:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00100000:1.0:1788205149.416869:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205149.416874:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933696, portal 25 00000100:00000001:3.0:1788205149.416878:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205149.416881:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 00000100:00000001:1.0:1788205149.416883:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205149.416887:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205149.416891:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205149.416895:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.416907:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788205149.416914:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205149.416914:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205149.416918:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205149.416923:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205933696, offset 0 00000100:00080000:1.0:1788205149.416929:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205149.416932:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:1.0:1788205149.416933:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205149.416936:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205149.416937:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205149.416940:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205149.416959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788205149.416965:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6604 00000400:00000200:3.0:1788205149.416968:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205149.416973:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.416977:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205149.416978:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205149.416982:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205149.416987:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205149.416987:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:1.0:1788205149.416991:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.416995:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.416997:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205149.416997:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1788205149.417002:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205149.417003:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205149.417005:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788205149.417005:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:1.0:1788205149.417008:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205149.417010:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205149.417014:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205149.417014:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59150:59150:256:4294967295] 192.168.201.151@tcp LPNI seq info [59130:59130:8:4294967295] 00000100:00000040:1.0:1788205149.417019:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417022:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205149.417025:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:1.0:1788205149.417028:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417032:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205149.417035:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205149.417043:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788205149.417044:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788205149.417049:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205149.417053:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1788205149.417056:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788205149.417057:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205149.417062:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205149.417063:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:1.0:1788205149.417064:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205149.417068:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205149.417070:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.417073:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205149.417074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000010:1.0:1788205149.417076:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788205149.417080:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:3.0:1788205149.417081:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 02000000:00000001:1.0:1788205149.417082:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205149.417084:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.417088:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205149.417089:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205149.417093:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205149.417093:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000010:1.0:1788205149.417100:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000800:00000200:3.1:1788205149.417100:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1788205149.417108:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205149.417108:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.417113:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00100000:1.0:1788205149.417118:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205149.417122:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205149.417132:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788205149.417133:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205149.417137:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205149.417138:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000040:1.0:1788205149.417143:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788205149.417144:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205149.417150:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788205149.417153:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205149.417155:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:3.0:1788205149.417158:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205149.417159:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205149.417161:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205149.417163:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205149.417167:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205149.417172:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6604 00000400:00000200:1.0:1788205149.417178:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205149.417179:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1788205149.417181:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205149.417184:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205149.417187:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205149.417188:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417192:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205149.417195:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417198:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205149.417201:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205149.417203:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788205149.417206:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205149.417207:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205149.417209:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205149.417211:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1788205149.417212:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205149.417215:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205149.417217:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417219:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205149.417219:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205149.417221:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417224:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205149.417225:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000200:3.0:1788205149.417227:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788205149.417229:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205149.417234:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417234:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301280769 02000000:00000001:1.0:1788205149.417239:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205149.417243:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205149.417243:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4000. 02000000:00000001:1.0:1788205149.417248:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.417250:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205149.417253:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205149.417253:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000008:00000001:1.0:1788205149.417259:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.417260:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205149.417264:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788205149.417266:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205149.417269:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205149.417270:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205149.417272:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205149.417274:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205149.417276:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417278:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301280769 02000000:00000010:1.0:1788205149.417280:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788205149.417285:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205149.417285:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f41e0. 00000100:00000001:3.0:1788205149.417292:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00100000:1.0:1788205149.417296:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205149.417299:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205149.417305:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417310:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788205149.417312:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788205149.417318:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205149/real 1788205149] req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205149.417327:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205149.417330:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205149.417333:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1788205149.417335:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788205149.417336:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205149.417342:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000001:1.0:1788205149.417345:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205149.417346:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000040:1.0:1788205149.417349:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6614) 00000100:00000001:3.0:1788205149.417350:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205149.417358:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205149.417387:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417394:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205149.417403:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417405:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.417407:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205149.417411:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417418:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205149.417422:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417429:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205149.417431:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417433:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.417434:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417436:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205149.417438:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205149.417440:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205149.417444:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205149.417446:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205149.417450:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205149.417458:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205933696:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205149.417462:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417463:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.417466:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205933696/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417472:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205149.417474:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205149.417477:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205149.417479:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205149.417481:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205149.417482:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205149.417484:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205149.417486:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:3.0:1788205149.417489:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205149.417491:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205149.417493:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000001:3.0:1788205149.417496:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205149.417497:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417499:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417505:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205149.417506:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417508:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205149.417509:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205149.417511:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205149.417514:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417521:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.417522:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.417524:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205149.417526:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205149.417528:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.417531:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933824:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205149.417534:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205149.417536:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205149.417537:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.417539:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205149.417542:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205149.417544:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205149.417547:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205149.417550:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205149.417553:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205149.417556:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933824, portal 10 00000100:00000001:3.0:1788205149.417558:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205149.417559:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:3.0:1788205149.417565:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417571:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205149.417573:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205149.417576:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205149.417578:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205933824, offset 0 00000400:00000200:3.0:1788205149.417590:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205149.417594:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.417598:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205149.417600:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205149.417603:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205149.417605:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205149.417607:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205149.417610:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59151:59151:256:4294967295] 192.168.201.151@tcp LPNI seq info [59131:59131:8:4294967295] 00000400:00000200:3.0:1788205149.417614:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.417620:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205149.417624:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.417628:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205149.417632:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205149.417635:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205149.417639:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205149.417642:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205149.417645:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.417647:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205149.417650:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205149.417653:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205149.417656:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205149.417658:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205149.417660:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205149.417663:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417671:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.417673:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417679:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417681:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205149.417684:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205149.417687:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205149.417691:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205149.417693:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301276827 00000100:00000001:3.0:1788205149.417695:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.417697:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301276827 00000100:00000001:3.0:1788205149.417699:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.417701:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205149.417703:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417705:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205149.417708:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205149/real 1788205149] req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205149.417721:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205149.417722:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205149.417724:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205149.417726:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205149.417728:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205149.417729:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205149.417732:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205149.417739:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417744:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205149.417745:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417747:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.417748:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205149.417752:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417758:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205149.417761:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417767:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205149.417770:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417771:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.417773:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.417774:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205149.417776:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205149.417779:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205149.417782:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205149.417784:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205149.417788:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205149.417795:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933824:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205149.417798:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205149.417802:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417808:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.417810:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.417812:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205149.417813:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205149.417815:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.417818:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933952:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205149.417821:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205149.417823:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205149.417824:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.417826:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205149.417829:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205149.417831:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205149.417834:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205149.417836:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205149.417839:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205149.417842:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205933952, portal 4 00000100:00000001:3.0:1788205149.417843:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205149.417845:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 00000100:00000040:3.0:1788205149.417850:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417856:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205149.417858:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205149.417861:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205149.417863:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205933952, offset 0 00000400:00000200:3.0:1788205149.417868:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205149.417871:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.417874:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205149.417877:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205149.417879:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205149.417881:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205149.417883:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205149.417886:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59152:59152:256:4294967295] 192.168.201.151@tcp LPNI seq info [59132:59132:8:4294967295] 00000400:00000200:3.0:1788205149.417890:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.417895:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205149.417900:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.417903:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205149.417906:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205149.417909:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205149.417913:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205149.417916:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205149.417919:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.417921:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205149.417923:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205149.417927:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205149.417929:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205149.417931:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205149.417933:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205149.417936:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417957:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.417961:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.417968:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417969:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205149.417973:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205149.417976:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205149.417979:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205149.417981:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301276713 00000100:00000001:3.0:1788205149.417984:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.417986:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301276713 00000100:00000001:3.0:1788205149.417988:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.417990:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205149.417992:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.417994:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205149.417998:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205149/real 1788205149] req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205149.418005:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205149.418006:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205149.418008:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205149.418010:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205149.418012:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205149.418014:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205149.418017:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205149.418024:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418030:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205149.418031:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418033:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.418034:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205149.418038:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418044:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205149.418047:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418053:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205149.418055:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418056:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.418057:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418059:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205149.418061:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205149.418062:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205149.418065:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205149.418067:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205149.418071:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205149.418078:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205933952:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205149.418082:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205149.418085:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418090:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.418092:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.418094:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205149.418095:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205149.418097:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.418100:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934080:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205149.418103:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205149.418105:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205149.418106:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205149.418108:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205149.418111:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205149.418113:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205149.418116:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205149.418118:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205149.418121:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205149.418124:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934080, portal 4 00000100:00000001:3.0:1788205149.418126:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205149.418127:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000040:3.0:1788205149.418132:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418138:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205149.418140:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205149.418142:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205149.418145:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205934080, offset 0 00000400:00000200:3.0:1788205149.418148:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205149.418152:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.418155:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205149.418158:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205149.418160:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205149.418163:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205149.418165:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205149.418167:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59153:59153:256:4294967295] 192.168.201.151@tcp LPNI seq info [59133:59133:8:4294967295] 00000400:00000200:3.0:1788205149.418171:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205149.418176:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205149.418180:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.418183:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205149.418187:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205149.418189:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205149.418193:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205149.418196:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205149.418199:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205149.418201:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205149.418204:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205149.418207:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205149.418209:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205149.418211:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205149.418213:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205149.418216:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418229:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.418231:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205174 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418237:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.418239:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205149.418241:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205149.418244:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205149.418247:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205149.418248:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301276719 00000100:00000001:3.0:1788205149.418251:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.418253:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301276719 00000100:00000001:3.0:1788205149.418255:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301280769 00000100:00000001:3.0:1788205149.418257:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205149.418259:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.418260:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205149.418263:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205149/real 1788205149] req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205149.418270:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205149.418271:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205149.418272:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205149.418275:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205149.418276:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205149.418278:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205149.418281:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205149.418289:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418295:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205149.418296:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418298:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.418299:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205149.418302:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418309:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205149.418312:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418318:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205149.418320:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418321:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205149.418323:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418325:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205149.418327:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205149.418329:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205149.418332:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205149.418334:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205149.418337:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205149.418345:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934080:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205149.418348:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418350:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.418353:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205933824/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418358:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205149.418360:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205149.418362:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205149.418381:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205149.418383:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205149.418384:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205149.418386:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205149.418388:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:3.0:1788205149.418390:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205149.418391:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205149.418393:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:3.0:1788205149.418395:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205149.418396:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418398:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.418401:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205933952/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418407:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205149.418409:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205149.418410:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205149.418412:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205149.418413:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205149.418414:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205149.418416:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205149.418417:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:3.0:1788205149.418419:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205149.418420:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205149.418422:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:3.0:1788205149.418424:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205149.418425:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418426:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205149.418430:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205934080/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205174 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205149.418436:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205149.418437:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205149.418438:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205149.418440:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205149.418441:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205149.418442:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205149.418444:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205149.418445:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:3.0:1788205149.418447:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205149.418449:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205149.418450:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00000001:3.0:1788205149.418452:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205149.418453:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418455:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.418458:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205149.418460:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.418462:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205149.418463:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205149.608499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205149.608504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205149.608509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205149.608509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205149.608511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205149.608512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205149.608514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205149.608515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205149.608561:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205149.608564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205149.608566:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205149.928457:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205149.928462:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205149.928464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.928468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205149.928470:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205149.928473:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.928475:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.928477:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.928478:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205150.376481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205150.376488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205150.376490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205150.376495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205150.376499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205150.376502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.376504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205150.376507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.376509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205150.440424:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205150.440431:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205150.440433:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205150.440439:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205150.440442:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205150.440446:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205150.440448:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205150.440451:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205150.440454:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205150.440537:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205150.440540:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205150.440541:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205150.440543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205150.440545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205150.440547:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.440549:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205150.440550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.440552:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205150.632461:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205150.632468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205150.632473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205150.632475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205150.632478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205150.632480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205150.632483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205150.632484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205150.632554:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205150.632558:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205150.632562:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205150.952511:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205150.952519:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205150.952521:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205150.952528:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205150.952531:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205150.952534:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.952536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205150.952538:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.952541:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.400511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205151.400520:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205151.400522:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.400526:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205151.400529:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205151.400533:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.400535:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.400537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.400539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.464519:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205151.464520:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205151.464528:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205151.464529:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205151.464532:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.464533:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205151.464539:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205151.464540:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205151.464545:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205151.464546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205151.464550:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.464550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205151.464555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205151.464558:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205151.464560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.464564:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.464568:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.464571:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205151.656520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205151.656534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205151.656537:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205151.656539:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656542:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205151.656544:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656545:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656547:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205151.656550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205151.656552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205151.656553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205151.656598:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205151.656601:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205151.656604:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205151.976473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205151.976480:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205151.976481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205151.976486:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205151.976489:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205151.976492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205151.976495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205151.976498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205151.976502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205152.424471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205152.424479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205152.424482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205152.424487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205152.424491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205152.424494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.424497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205152.424499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.424503:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205152.488473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205152.488481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205152.488483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205152.488488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205152.488492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205152.488496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.488499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205152.488501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.488504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205152.489058:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205152.489061:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205152.489063:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205152.489065:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205152.489068:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205152.489070:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205152.489072:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205152.489073:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205152.489075:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205152.680772:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205152.680778:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205152.680783:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205152.680785:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205152.680788:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205152.680789:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205152.680792:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205152.680793:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205152.680854:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205152.680857:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205152.680873:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205153.000529:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205153.000539:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205153.000541:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205153.000548:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205153.000554:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205153.000559:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205153.000562:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205153.000565:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205153.000569:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.448490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205153.448495:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205153.448497:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205153.448500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205153.448503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205153.448506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.448507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205153.448509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.448511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205153.512473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205153.512473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205153.512481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205153.512481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205153.512484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.512485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205153.512490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205153.512490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205153.512494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205153.512496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205153.512497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.512500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205153.512501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205153.512503:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205153.512506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.512507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205153.512510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.512513:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205153.704536:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205153.704538:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205153.704541:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704544:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205153.704546:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704547:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704549:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205153.704552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205153.704555:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205153.704556:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205153.704557:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704558:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205153.704559:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704560:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704561:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205153.704564:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205153.704564:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205153.704565:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704566:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205153.704567:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704568:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205153.704570:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205153.704648:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205153.704655:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205153.704659:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205154.024497:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301280769, at 4301281280 00000100:00000010:1.0:1788205154.024510:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000040:0.0:1788205154.024598:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301280769, at 4301281280 00000100:00000010:0.0:1788205154.024612:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000001:3.0:1788205154.024629:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205154.024634:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.024637:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.024641:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205154.024645:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205154.024648:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.024651:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.024653:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.024655:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.472767:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205154.472778:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.472781:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.472789:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205154.472793:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205154.472797:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.472800:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.472804:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.472808:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205154.536484:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205154.536495:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205154.536497:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205154.536501:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205154.536506:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205154.536511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788205154.536514:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6609 00000100:00000001:3.0:1788205154.536516:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.536520:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.536524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000400:00000200:1.0:1788205154.536525:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205154.536529:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1788205154.536534:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205154.536535:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205154.536540:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205154.536541:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.536545:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205154.536547:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.536550:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.536552:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205154.536557:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.536559:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205154.536563:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205154.536566:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205154.536568:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205154.536571:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205154.536574:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205154.536577:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205154.536580:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.536582:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205154.536584:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.536586:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205154.536589:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205154.536592:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205154.536594:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205154.536598:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205154.536601:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205154.536604:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.536606:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205154.536608:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205154.536617:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534380. 00000020:00000040:1.0:1788205154.536621:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205154.536625:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205154.536627:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.536630:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205154.536632:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205154.536636:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205154.536644:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be400. 00000100:00000001:1.0:1788205154.536650:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205154.536659:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205154.536668:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205154.536672:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205154.536676:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea578df6] to pc [ptlrpcd_rcv+-1] req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205154.536697:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205154.536700:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205154.536703:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205154.536704:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205154.536705:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205154.536708:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205154.536711:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6609 00000400:00000200:1.0:1788205154.536715:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205154.536720:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205154.536722:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205154.536723:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788205154.536727:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.536729:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205154.536731:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205154.536733:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205154.536737:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.536742:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:3.0:1788205154.536743:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205154.536747:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205154.536750:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205154.536754:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205154.536758:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:3.0:1788205154.536758:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.536761:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205154.536763:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788205154.536766:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1788205154.536768:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788205154.536770:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205154.536775:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:3.0:1788205154.536775:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.536781:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788205154.536783:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205934208:192.168.201.151@tcp:250:kworker.0 00000100:00000040:1.0:1788205154.536786:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.536790:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205154.536791:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:3.0:1788205154.536795:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1788205154.536796:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205154.536800:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205154.536802:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788205154.536804:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788205154.536806:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205154.536810:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1788205154.536811:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:1.0:1788205154.536815:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205154.536815:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205154.536819:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.536822:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205154.536827:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205154.536833:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205154.536840:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934208, portal 25 00000100:00000001:3.0:1788205154.536843:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205154.536844:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292777856 : -113511416773760 : ffff98c310534380) 00000100:00000040:3.0:1788205154.536850:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.536874:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205154.536878:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205154.536881:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205154.536885:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205934208, offset 0 00000400:00000200:3.0:1788205154.536891:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205154.536899:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.536904:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205154.536909:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205154.536914:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205154.536917:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205154.536920:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205154.536924:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59154:59154:256:4294967295] 192.168.201.151@tcp LPNI seq info [59134:59134:8:4294967295] 00000400:00000200:3.0:1788205154.536928:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.536938:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205154.536943:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.536949:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205154.536956:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205154.536960:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205154.536972:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205154.536976:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788205154.536977:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205154.536982:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000800:00000100:3.0:1788205154.536982:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205154.536986:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205154.536987:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00100000:2.0:1788205154.536991:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205154.536995:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1788205154.536996:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205154.536998:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205154.537002:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1788205154.537004:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205154.537005:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205154.537010:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.537012:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205154.537016:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205154.537021:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205154.537025:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537034:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.537037:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205154.537047:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205154.537053:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205154.537054:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537059:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:1.0:1788205154.537060:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537064:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537067:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205154.537068:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535f80. 00000020:00000040:1.0:1788205154.537073:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:3.0:1788205154.537074:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788205154.537079:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205154.537080:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301285889 02000000:00000001:1.0:1788205154.537082:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537085:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205154.537087:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:3.0:1788205154.537088:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 02000000:00000001:1.0:1788205154.537090:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537096:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000010:1.0:1788205154.537097:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf800. 00000100:00000001:1.0:1788205154.537103:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537105:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301285889 00000100:00100000:1.0:1788205154.537111:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788205154.537111:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f48a0. 00000100:00000001:3.0:1788205154.537117:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205154.537123:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205154.537125:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205154.537128:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537129:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537133:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788205154.537137:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8c535f6] to pc [ptlrpcd_rcv+-1] req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205154.537141:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205154/real 1788205154] req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205154.537162:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205154.537164:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1788205154.537166:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:3.0:1788205154.537168:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205154.537171:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00080000:1.0:1788205154.537173:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788205154.537175:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000001:1.0:1788205154.537176:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788205154.537180:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00080000:1.0:1788205154.537181:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205154.537185:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1788205154.537186:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205154.537191:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205154.537194:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6609 00000400:00000200:1.0:1788205154.537204:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:3.0:1788205154.537210:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205154.537212:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205154.537217:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205154.537225:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.537228:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205154.537230:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205154.537232:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205154.537235:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.537237:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205154.537240:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205154.537241:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788205154.537247:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205154.537251:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00100000:3.0:1788205154.537251:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205154.537253:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205154.537257:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205154.537263:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:3.0:1788205154.537266:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788205154.537267:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205154.537274:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788205154.537275:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205154.537279:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205154.537284:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537289:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205154.537290:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205154.537294:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.537295:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.537297:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.537299:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205154.537303:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:1.0:1788205154.537304:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205154.537309:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205154.537311:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537316:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205154.537317:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 02000000:00000001:1.0:1788205154.537319:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205154.537321:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1788205154.537324:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788205154.537328:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205154.537329:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537334:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205154.537338:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205154.537344:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.537344:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205934208:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205154.537350:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537353:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1788205154.537355:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534000. 00000100:00000001:3.0:1788205154.537357:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1788205154.537360:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205154.537382:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205154.537383:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537385:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205154.537387:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205154.537389:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.537390:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534380 x1875062205934208/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788205154.537394:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102be000. 00000100:00000001:1.0:1788205154.537401:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537403:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205154.537408:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1788205154.537409:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1788205154.537412:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205154.537418:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205154.537421:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:1.0:1788205154.537422:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000040:3.0:1788205154.537425:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537426:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205154.537430:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:1.0:1788205154.537435:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006a0b779] to pc [ptlrpcd_rcv+-1] req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788205154.537435:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be400. 02000000:00000001:3.0:1788205154.537441:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205154.537447:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788205154.537448:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205154.537450:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534380. 00000100:00080000:1.0:1788205154.537452:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205154.537455:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1788205154.537457:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205154.537459:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205154.537462:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205154.537463:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205154.537465:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205154.537470:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205154.537473:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6609 00000100:00000001:3.0:1788205154.537473:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.537476:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205154.537481:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205154.537481:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205154.537484:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205154.537487:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1788205154.537488:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205154.537492:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205154.537494:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205154.537498:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537503:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205154.537507:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205154.537510:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205154.537511:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205154.537513:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205154.537515:0:12771:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788205154.537518:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1788205154.537519:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 02000000:00000001:3.0:1788205154.537521:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205154.537525:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205154.537528:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1788205154.537530:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934336:192.168.201.151@tcp:38:kworker.0 00000020:00000040:1.0:1788205154.537532:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205154.537537:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1788205154.537538:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 02000000:00000001:3.0:1788205154.537540:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1788205154.537542:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 02000000:00000001:3.0:1788205154.537544:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205154.537547:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1788205154.537549:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000010:3.0:1788205154.537552:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 00000100:00000001:1.0:1788205154.537553:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205154.537556:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205154.537558:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537563:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205154.537563:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:1.0:1788205154.537568:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000010:3.0:1788205154.537569:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00080000:1.0:1788205154.537576:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:3.0:1788205154.537576:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:1.0:1788205154.537582:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205154.537582:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934336, portal 10 00000100:00000001:3.0:1788205154.537586:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1788205154.537587:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205154.537589:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785024 : -113511416766592 : ffff98c310535f80) 02000000:00000001:1.0:1788205154.537592:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205154.537596:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.537598:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205154.537600:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205154.537603:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205154.537610:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537611:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788205154.537613:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.537618:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000010:1.0:1788205154.537620:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537800. 00000020:00000040:1.0:1788205154.537624:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205154.537627:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788205154.537628:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 02000000:00000001:1.0:1788205154.537629:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205154.537633:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1788205154.537633:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205934336, offset 0 02000000:00000001:1.0:1788205154.537636:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205154.537640:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.537642:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000010:1.0:1788205154.537646:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf400. 00000400:00000200:3.0:1788205154.537648:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205154.537652:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.537654:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205154.537659:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:1.0:1788205154.537662:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205154.537666:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205154.537671:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205154.537677:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:1.0:1788205154.537680:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205154.537682:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59155:59155:256:4294967295] 192.168.201.151@tcp LPNI seq info [59135:59135:8:4294967295] 00000100:00000001:1.0:1788205154.537684:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.537690:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:1.0:1788205154.537694:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000be56f33e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205154.537706:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788205154.537715:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205154.537716:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.537721:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000040:1.0:1788205154.537722:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6619) 00000800:00000200:3.0:1788205154.537728:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205154.537732:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205154.537736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205154.537739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205154.537743:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.537744:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205154.537748:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205154.537752:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205154.537755:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205154.537757:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205154.537759:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205154.537762:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537770:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.537773:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537779:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537780:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205154.537784:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.537788:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205154.537791:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205154.537793:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301281947 00000100:00000001:3.0:1788205154.537795:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.537798:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301281947 00000100:00000001:3.0:1788205154.537800:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.537802:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205154.537803:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.537805:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205154.537808:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205154/real 1788205154] req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205154.537815:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205154.537817:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205154.537819:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205154.537821:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205154.537822:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205154.537824:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205154.537827:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205154.537835:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537841:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205154.537842:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.537843:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.537845:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205154.537848:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537884:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205154.537889:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537896:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205154.537898:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.537900:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.537901:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.537903:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205154.537905:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205154.537907:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205154.537910:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205154.537912:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205154.537915:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205154.537922:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934336:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205154.537925:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205154.537928:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537934:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.537936:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205154.537938:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205154.537939:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205154.537941:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.537944:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934464:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205154.537946:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205154.537948:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205154.537949:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205154.537951:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205154.537954:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205154.537956:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.537959:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205154.537963:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205154.537965:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205154.537968:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934464, portal 4 00000100:00000001:3.0:1788205154.537969:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205154.537970:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292776960 : -113511416774656 : ffff98c310534000) 00000100:00000040:3.0:1788205154.537975:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.537980:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205154.537982:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205154.537984:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205154.537986:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205934464, offset 0 00000400:00000200:3.0:1788205154.537990:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205154.537993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.537996:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205154.537999:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205154.538001:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205154.538003:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205154.538005:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205154.538008:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59156:59156:256:4294967295] 192.168.201.151@tcp LPNI seq info [59136:59136:8:4294967295] 00000400:00000200:3.0:1788205154.538011:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.538017:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205154.538021:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.538024:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205154.538027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205154.538030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205154.538034:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205154.538036:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205154.538039:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.538041:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205154.538044:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205154.538047:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205154.538049:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205154.538051:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205154.538053:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205154.538057:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538064:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.538066:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538072:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538073:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205154.538076:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.538079:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205154.538082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205154.538083:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301281833 00000100:00000001:3.0:1788205154.538086:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.538088:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301281833 00000100:00000001:3.0:1788205154.538090:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.538092:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205154.538094:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538095:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205154.538098:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205154/real 1788205154] req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205154.538105:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205154.538106:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205154.538108:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205154.538110:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205154.538111:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205154.538113:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205154.538116:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205154.538124:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538134:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205154.538136:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538137:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.538138:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205154.538142:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538148:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205154.538151:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538157:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205154.538159:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538161:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.538162:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538163:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205154.538165:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205154.538167:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205154.538169:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205154.538171:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205154.538175:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205154.538181:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934464:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205154.538184:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538186:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.538188:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535f80 x1875062205934336/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538193:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205154.538195:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205154.538197:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205154.538199:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205154.538200:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205154.538202:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205154.538203:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205154.538205:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf800. 02000000:00000001:3.0:1788205154.538207:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205154.538208:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205154.538209:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535f80. 00000100:00000001:3.0:1788205154.538212:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205154.538213:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538215:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.538218:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534000 x1875062205934464/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538224:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205154.538225:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205154.538226:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205154.538228:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205154.538229:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205154.538230:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205154.538232:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205154.538233:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102be000. 02000000:00000001:3.0:1788205154.538235:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205154.538236:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205154.538238:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534000. 00000100:00000001:3.0:1788205154.538240:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205154.538241:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538242:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538245:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.538247:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538248:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205154.538250:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205154.538251:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205154.538255:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538261:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.538263:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205154.538265:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205154.538267:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205154.538269:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205154.538271:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205154.538274:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205154.538276:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205154.538277:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205154.538278:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205154.538281:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205154.538283:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205154.538286:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205154.538288:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205154.538291:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205154.538293:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934592, portal 4 00000100:00000001:3.0:1788205154.538295:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205154.538296:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292791296 : -113511416760320 : ffff98c310537800) 00000100:00000040:3.0:1788205154.538301:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538307:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205154.538309:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205154.538310:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205154.538313:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205934592, offset 0 00000400:00000200:3.0:1788205154.538316:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205154.538319:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.538322:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205154.538325:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205154.538327:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205154.538329:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205154.538331:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205154.538334:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59157:59157:256:4294967295] 192.168.201.151@tcp LPNI seq info [59137:59137:8:4294967295] 00000400:00000200:3.0:1788205154.538337:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205154.538343:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205154.538348:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.538350:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205154.538354:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205154.538357:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205154.538361:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205154.538380:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205154.538384:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205154.538386:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205154.538389:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205154.538392:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205154.538394:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205154.538396:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205154.538398:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205154.538401:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538407:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.538411:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538416:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538417:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205154.538420:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205154.538422:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205154.538425:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205154.538427:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301281840 00000100:00000001:3.0:1788205154.538429:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.538431:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301281840 00000100:00000001:3.0:1788205154.538433:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301285889 00000100:00000001:3.0:1788205154.538435:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205154.538437:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538438:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205154.538441:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205154/real 1788205154] req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205154.538449:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205154.538451:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205154.538452:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205154.538454:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205154.538456:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205154.538457:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205154.538460:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205154.538467:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538473:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205154.538474:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538476:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.538477:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205154.538480:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538486:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205154.538490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538496:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205154.538498:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538499:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205154.538501:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538503:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205154.538504:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205154.538506:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205154.538509:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205154.538511:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205154.538515:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205154.538523:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934592:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205154.538526:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538528:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205154.538530:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537800 x1875062205934592/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205154.538535:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205154.538537:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205154.538538:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205154.538540:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205154.538542:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205154.538543:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205154.538544:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205154.538546:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf400. 02000000:00000001:3.0:1788205154.538548:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205154.538549:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205154.538550:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537800. 00000100:00000001:3.0:1788205154.538552:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205154.538553:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538555:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.538559:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205154.538560:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.538561:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205154.538562:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205154.728482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205154.728490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205154.728497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205154.728498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205154.728501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205154.728503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205154.728507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205154.728508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205154.728582:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205154.728586:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205154.728589:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205155.048489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205155.048496:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205155.048498:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205155.048503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205155.048507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205155.048511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.048513:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205155.048516:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.048518:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205155.496481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205155.496488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205155.496490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205155.496496:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205155.496499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205155.496502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.496505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205155.496508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.496511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205155.560478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205155.560478:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205155.560484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205155.560485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205155.560486:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205155.560489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205155.560490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205155.560495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205155.560496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205155.560498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205155.560501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205155.560501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205155.560505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.560506:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205155.560507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205155.560511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205155.560515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.560517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205155.752511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205155.752518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205155.752528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205155.752530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205155.752534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205155.752535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205155.752538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205155.752539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205155.752602:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205155.752606:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205155.752609:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205156.072469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205156.072475:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.072478:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.072482:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205156.072486:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205156.072490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.072493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.072496:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.072499:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.520481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205156.520487:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.520489:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.520493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205156.520496:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205156.520498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.520500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.520502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.520504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.584468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205156.584468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205156.584476:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.584476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205156.584479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.584480:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205156.584487:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205156.584487:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205156.584492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205156.584492:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205156.584498:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.584499:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205156.584504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205156.584508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205156.584511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.584511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.584515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.584519:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205156.776491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205156.776498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205156.776504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205156.776505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205156.776507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205156.776509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205156.776511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205156.776512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205156.776577:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205156.776581:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205156.776584:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205157.096473:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205157.096479:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205157.096480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205157.096484:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205157.096487:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205157.096490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.096492:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205157.096493:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.096495:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205157.544452:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205157.544459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205157.544462:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205157.544468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205157.544471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205157.544474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.544477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205157.544479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.544482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205157.608477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205157.608477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205157.608485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205157.608485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205157.608488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205157.608489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205157.608495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:2.0:1788205157.608496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205157.608500:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205157.608501:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205157.608505:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205157.608506:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.608509:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205157.608512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.608515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205157.608516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205157.608519:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205157.608522:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205157.800524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205157.800530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205157.800543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205157.800544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205157.800548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205157.800549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205157.800552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205157.800553:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205157.800609:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205157.800613:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205157.800616:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205158.120483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205158.120488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205158.120490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205158.120493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205158.120496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205158.120499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.120501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205158.120504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.120506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205158.568479:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205158.568486:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205158.568488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205158.568495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205158.568498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205158.568502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.568504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205158.568506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.568509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205158.632470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205158.632482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205158.632484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205158.632490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205158.632493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205158.632497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205158.632499:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205158.632501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205158.632504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205158.632577:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205158.632579:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205158.632581:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205158.632583:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205158.632585:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205158.632587:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.632589:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205158.632590:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.632592:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205158.824473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205158.824479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205158.824485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205158.824486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205158.824489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205158.824490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205158.824493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205158.824494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205158.824556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205158.824560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205158.824563:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205159.144490:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301285889, at 4301286400 00000100:00000010:0.0:1788205159.144507:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:3.0:1788205159.144599:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205159.144610:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.144613:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.144618:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205159.144621:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205159.144624:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.144628:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.144631:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.144634:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205159.144674:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301285889, at 4301286400 00000100:00000010:1.0:1788205159.144681:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000001:3.0:1788205159.592511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205159.592521:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.592523:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.592531:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205159.592535:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205159.592540:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.592544:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.592547:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.592551:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.656501:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1788205159.656508:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788205159.656510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205159.656514:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205159.656521:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.656521:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205159.656522:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205159.656528:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.656529:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205159.656531:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:0.0:1788205159.656534:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788205159.656538:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205159.656538:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205159.656540:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205159.656543:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.656550:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205159.656551:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205159.656554:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6614 00000100:00000001:3.0:1788205159.656554:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205159.656559:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.656560:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205159.656568:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205159.656572:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205159.656576:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788205159.656579:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:2.0:1788205159.656582:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205159.656591:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205159.656595:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656598:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205159.656600:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656601:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205159.656605:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205159.656609:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205159.656612:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205159.656616:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205159.656618:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205159.656620:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205159.656623:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656624:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205159.656625:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656626:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205159.656628:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205159.656630:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656632:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205159.656634:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205159.656638:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205159.656641:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.656644:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205159.656646:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205159.656657:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205159.656661:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205159.656665:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205159.656667:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.656670:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205159.656672:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205159.656677:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205159.656686:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788205159.656695:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205159.656705:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205159.656713:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205159.656716:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205159.656720:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205159.656750:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205159.656753:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205159.656763:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205159.656765:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205159.656779:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205159.656783:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205159.656786:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6614 00000400:00000200:0.0:1788205159.656791:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205159.656793:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1788205159.656795:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205159.656797:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205159.656799:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205159.656800:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205159.656804:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656808:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205159.656808:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205159.656811:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656816:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205159.656820:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788205159.656821:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788205159.656822:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205159.656824:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 02000000:00000001:3.0:1788205159.656826:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788205159.656827:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788205159.656830:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1788205159.656831:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:3.0:1788205159.656833:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205159.656834:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205159.656839:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00100000:3.0:1788205159.656839:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205934720:192.168.201.151@tcp:250:kworker.0 00000100:00000040:0.0:1788205159.656846:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656849:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205159.656851:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.656853:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205159.656855:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205159.656858:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205159.656860:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788205159.656861:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205159.656865:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205159.656867:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205159.656872:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205159.656872:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.656876:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788205159.656877:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.656880:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:3.0:1788205159.656884:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 00000008:00100000:0.0:1788205159.656887:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205159.656888:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788205159.656894:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.656898:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.656899:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:0.0:1788205159.656905:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000400:00000010:3.0:1788205159.656906:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000020:00000040:0.0:1788205159.656910:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205159.656914:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205159.656916:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205159.656916:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:0.0:1788205159.656919:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205159.656922:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205159.656923:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205159.656925:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934720, portal 25 00000100:00000001:3.0:1788205159.656929:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:0.0:1788205159.656930:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:3.0:1788205159.656931:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000001:0.0:1788205159.656937:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205159.656939:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205159.656946:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205159.656948:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205159.656952:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205159.656955:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205159.656959:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205934720, offset 0 00000100:00000001:0.0:1788205159.656961:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205159.656965:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.656966:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788205159.656973:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205159.656978:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.656986:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:0.0:1788205159.656987:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205159.656992:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205159.656994:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1788205159.656999:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788205159.657000:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:0.0:1788205159.657004:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788205159.657006:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1788205159.657008:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205159.657009:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657014:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205159.657015:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59158:59158:256:4294967295] 192.168.201.151@tcp LPNI seq info [59138:59138:8:4294967295] 00000400:00000200:3.0:1788205159.657020:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788205159.657027:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6614 00000400:00000200:3.0:1788205159.657029:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000400:00000200:0.0:1788205159.657039:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:3.0:1788205159.657039:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205159.657046:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000010:3.0:1788205159.657047:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00080000:0.0:1788205159.657052:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205159.657055:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657059:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788205159.657060:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205159.657067:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205159.657074:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:0.0:1788205159.657080:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657083:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205159.657086:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205159.657087:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205159.657089:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205159.657091:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:0.0:1788205159.657097:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000200:3.1:1788205159.657097:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000020:00000001:0.0:1788205159.657101:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000800:00000100:3.0:1788205159.657107:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788205159.657108:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000010:3.0:1788205159.657111:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000001:0.0:1788205159.657113:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205159.657123:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657126:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205159.657126:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:0.0:1788205159.657131:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:3.0:1788205159.657135:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1788205159.657138:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:3.0:1788205159.657141:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00000001:0.0:1788205159.657143:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205159.657144:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 02000000:00000001:0.0:1788205159.657149:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205159.657151:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1788205159.657154:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1788205159.657156:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205159.657159:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.657164:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205159.657168:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:0.0:1788205159.657169:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000040:3.0:1788205159.657172:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788205159.657174:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.657179:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657184:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205159.657185:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205159.657186:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1788205159.657191:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:3.0:1788205159.657192:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205159.657194:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205159.657198:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.657198:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205159.657203:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205159.657204:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301291009 02000000:00000001:0.0:1788205159.657206:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:3.0:1788205159.657209:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4de0. 02000000:00000001:0.0:1788205159.657210:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205159.657216:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 00000100:00000001:3.0:1788205159.657217:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205159.657220:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657225:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301291009 00000100:00100000:0.0:1788205159.657229:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788205159.657230:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f40c0. 00000100:00000001:3.0:1788205159.657234:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205159.657237:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205159.657241:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.657245:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205159.657245:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205159.657249:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.657251:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205159/real 1788205159] req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205159.657257:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205159.657261:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205159.657263:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205159.657266:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000100:00000001:0.0:1788205159.657267:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.657269:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00080000:0.0:1788205159.657270:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788205159.657271:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00080000:0.0:1788205159.657276:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205159.657276:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205159.657279:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205159.657283:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788205159.657283:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205159.657291:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205159.657294:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6614 00000100:00000200:3.0:1788205159.657294:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205159.657303:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205159.657304:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205159.657306:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788205159.657307:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205159.657308:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657311:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788205159.657312:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205159.657316:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205159.657319:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657325:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205159.657326:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205159.657330:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:3.0:1788205159.657333:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205159.657335:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205159.657340:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1788205159.657342:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788205159.657343:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788205159.657346:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:3.0:1788205159.657346:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657348:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205159.657351:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205159.657352:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205159.657354:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205159.657356:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:0.0:1788205159.657358:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00080000:3.0:1788205159.657359:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000020:00000001:0.0:1788205159.657392:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205159.657396:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788205159.657397:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205159.657402:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657406:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205159.657408:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205159.657412:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205159.657417:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788205159.657422:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205934720:192.168.201.151@tcp:250:kworker.0 00000020:00000040:0.0:1788205159.657423:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205159.657430:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205159.657430:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657435:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205159.657436:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205159.657440:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1788205159.657441:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205934720/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205159.657445:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205159.657450:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657454:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1788205159.657455:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205159.657459:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:0.0:1788205159.657460:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 02000000:00000010:3.0:1788205159.657463:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 00000008:00000001:0.0:1788205159.657465:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.657468:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657469:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205159.657472:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000010:0.0:1788205159.657473:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:3.0:1788205159.657474:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000040:0.0:1788205159.657478:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205159.657479:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205159.657483:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205159.657484:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205159.657487:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205159.657490:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205159.657490:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205159.657494:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205159.657496:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205159.657500:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788205159.657504:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657504:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205159.657508:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00100000:0.0:1788205159.657513:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205159.657513:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205159.657516:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657520:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205159.657528:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205159.657530:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657531:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.657534:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205159.657537:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657537:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205159.657540:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205159.657543:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205159.657549:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205159.657551:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205159.657558:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6624) 00000100:00000001:3.0:1788205159.657560:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.657563:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657565:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205159.657568:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205159.657570:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.657573:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934848:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205159.657577:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205159.657579:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205159.657581:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657582:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205159.657586:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205159.657588:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.657592:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205159.657594:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205159.657598:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205159.657600:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934848, portal 10 00000100:00000001:3.0:1788205159.657602:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205159.657603:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:3.0:1788205159.657610:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657616:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205159.657619:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205159.657622:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205159.657625:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205934848, offset 0 00000400:00000200:3.0:1788205159.657628:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205159.657631:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.657633:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205159.657636:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205159.657637:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205159.657639:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205159.657641:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205159.657643:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59159:59159:256:4294967295] 192.168.201.151@tcp LPNI seq info [59139:59139:8:4294967295] 00000400:00000200:3.0:1788205159.657645:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.657650:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205159.657652:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.657655:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205159.657657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205159.657659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205159.657662:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205159.657664:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205159.657666:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.657668:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205159.657670:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205159.657672:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205159.657674:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205159.657675:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205159.657676:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205159.657679:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657684:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.657686:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657690:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657691:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205159.657693:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205159.657696:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205159.657698:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205159.657700:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301287067 00000100:00000001:3.0:1788205159.657701:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.657703:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301287067 00000100:00000001:3.0:1788205159.657704:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.657705:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205159.657706:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657708:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205159.657710:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205159/real 1788205159] req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205159.657714:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205159.657715:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205159.657716:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205159.657717:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205159.657718:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205159.657719:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205159.657721:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205159.657726:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657730:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205159.657731:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657732:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657732:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205159.657734:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657738:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205159.657740:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657744:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205159.657746:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657747:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657748:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657749:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205159.657750:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205159.657752:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205159.657753:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205159.657754:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205159.657756:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205159.657760:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934848:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205159.657762:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205159.657764:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657780:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.657781:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657782:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205159.657784:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205159.657785:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.657787:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934976:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205159.657789:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205159.657791:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205159.657792:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.657793:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205159.657795:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205159.657797:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.657799:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205159.657801:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205159.657803:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205159.657805:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205934976, portal 4 00000100:00000001:3.0:1788205159.657807:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205159.657807:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:3.0:1788205159.657811:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657815:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205159.657817:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205159.657818:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205159.657820:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205934976, offset 0 00000400:00000200:3.0:1788205159.657821:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205159.657823:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.657825:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205159.657827:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205159.657829:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205159.657830:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205159.657832:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205159.657834:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59160:59160:256:4294967295] 192.168.201.151@tcp LPNI seq info [59140:59140:8:4294967295] 00000400:00000200:3.0:1788205159.657836:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.657840:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205159.657842:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.657844:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205159.657846:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205159.657848:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205159.657851:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205159.657852:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205159.657855:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.657856:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205159.657858:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205159.657860:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205159.657862:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205159.657863:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205159.657865:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205159.657867:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657871:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.657873:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657877:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657878:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205159.657880:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205159.657882:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205159.657883:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205159.657885:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301286953 00000100:00000001:3.0:1788205159.657886:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.657888:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301286953 00000100:00000001:3.0:1788205159.657889:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.657890:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205159.657891:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657892:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205159.657894:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205159/real 1788205159] req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205159.657898:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205159.657899:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205159.657900:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205159.657902:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205159.657902:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205159.657903:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205159.657905:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205159.657909:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657917:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205159.657918:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657919:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657920:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205159.657922:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657925:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205159.657927:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657931:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205159.657932:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657933:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.657933:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657934:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205159.657936:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205159.657938:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205159.657940:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205159.657941:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205159.657943:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205159.657947:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205934976:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205159.657949:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657950:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.657952:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205934848/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657956:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205159.657957:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205159.657959:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205159.657960:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205159.657961:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205159.657962:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205159.657964:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205159.657965:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:3.0:1788205159.657966:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205159.657967:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205159.657968:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:3.0:1788205159.657970:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205159.657970:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657971:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.657973:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205934976/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657977:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205159.657978:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205159.657979:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205159.657980:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205159.657980:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205159.657981:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205159.657982:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205159.657983:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:3.0:1788205159.657985:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205159.657985:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205159.657986:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:3.0:1788205159.657987:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205159.657988:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657989:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.657991:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.657991:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.657992:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205159.657993:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205159.657994:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205159.657996:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.657999:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.658000:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.658002:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205159.658003:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205159.658004:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205159.658005:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205159.658007:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205159.658008:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205159.658009:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205159.658010:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205159.658011:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205159.658012:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205159.658014:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205159.658015:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205159.658017:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205159.658018:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935104, portal 4 00000100:00000001:3.0:1788205159.658019:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205159.658020:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:3.0:1788205159.658024:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658028:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205159.658029:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205159.658030:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205159.658032:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205935104, offset 0 00000400:00000200:3.0:1788205159.658034:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205159.658035:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.658037:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205159.658039:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205159.658040:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205159.658041:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205159.658043:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205159.658045:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59161:59161:256:4294967295] 192.168.201.151@tcp LPNI seq info [59141:59141:8:4294967295] 00000400:00000200:3.0:1788205159.658047:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205159.658052:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205159.658056:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.658059:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205159.658063:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205159.658066:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205159.658070:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205159.658072:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205159.658076:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205159.658078:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205159.658081:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205159.658084:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205159.658086:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205159.658088:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205159.658090:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205159.658094:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658101:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.658104:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658108:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.658109:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205159.658112:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205159.658114:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205159.658116:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205159.658117:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301286959 00000100:00000001:3.0:1788205159.658119:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.658122:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301286959 00000100:00000001:3.0:1788205159.658124:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301291009 00000100:00000001:3.0:1788205159.658126:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205159.658128:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.658129:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205159.658132:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205159/real 1788205159] req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205159.658140:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205159.658142:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205159.658143:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205159.658146:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205159.658147:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205159.658149:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205159.658152:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205159.658160:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658166:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205159.658167:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658169:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.658171:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205159.658174:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658179:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205159.658183:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658189:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205159.658191:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658193:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205159.658194:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658196:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205159.658198:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205159.658200:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205159.658203:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205159.658206:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205159.658210:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205159.658216:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935104:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205159.658218:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658220:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205159.658221:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205935104/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205159.658225:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205159.658226:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205159.658227:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205159.658228:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205159.658229:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205159.658230:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205159.658232:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205159.658233:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:3.0:1788205159.658234:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205159.658235:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205159.658236:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:3.0:1788205159.658237:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205159.658238:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658239:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.658241:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.658242:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.658244:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205159.658245:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205159.848554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205159.848561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205159.848567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205159.848568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205159.848570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205159.848571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205159.848574:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205159.848575:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205159.848799:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205159.848803:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205159.848808:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205160.168521:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205160.168533:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205160.168535:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205160.168542:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205160.168546:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205160.168551:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.168555:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205160.168558:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.168561:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.616518:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205160.616527:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205160.616530:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205160.616537:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205160.616541:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205160.616546:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.616548:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205160.616556:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.616560:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205160.680512:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205160.680513:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205160.680526:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205160.680527:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205160.680534:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.680536:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205160.680549:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:2.0:1788205160.680550:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205160.680554:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205160.680562:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205160.680563:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205160.680569:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205160.680569:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205160.680574:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.680579:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205160.680580:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.680585:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205160.680589:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205160.872509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205160.872516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205160.872522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205160.872523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205160.872526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205160.872527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205160.872530:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205160.872531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205160.872647:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205160.872654:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205160.872659:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205161.192495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205161.192506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205161.192508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.192515:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205161.192518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.192522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.192525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.192528:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.192530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.640505:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205161.640515:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205161.640517:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.640524:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205161.640528:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.640533:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.640536:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.640540:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.640549:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.704532:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205161.704536:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205161.704543:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205161.704546:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205161.704552:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205161.704556:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205161.704560:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.704563:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205161.704564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205161.704568:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.704569:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.704574:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.704575:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205161.704581:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.704584:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.704586:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.704588:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.704591:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205161.896559:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205161.896567:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205161.896570:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205161.896573:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896583:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205161.896586:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896588:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896590:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205161.896596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205161.896599:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205161.896600:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205161.896658:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205161.896661:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205161.896664:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205162.216483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205162.216491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205162.216494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205162.216498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205162.216502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205162.216506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.216509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205162.216511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.216514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.664570:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205162.664577:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205162.664578:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205162.664582:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205162.664584:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205162.664588:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.664590:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205162.664592:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.664595:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.728477:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205162.728482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205162.728484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205162.728487:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788205162.728489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205162.728493:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205162.728493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205162.728497:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.728498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205162.728503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205162.728503:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205162.728505:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205162.728508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205162.728510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205162.728512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205162.728525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205162.728527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205162.728530:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205162.920495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205162.920502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205162.920508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205162.920509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205162.920513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205162.920515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205162.920519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205162.920520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205162.920605:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205162.920610:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205162.920614:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205163.240480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205163.240487:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205163.240489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205163.240494:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205163.240498:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205163.240502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.240505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205163.240508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.240511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.688471:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205163.688479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205163.688482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205163.688487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205163.688490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205163.688494:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.688496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205163.688498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.688501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.752499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205163.752508:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205163.752510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205163.752515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205163.752518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205163.752522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.752533:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205163.752536:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205163.752539:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205163.752552:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205163.752591:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205163.752593:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205163.752597:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205163.752600:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205163.752602:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205163.752604:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205163.752606:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205163.752608:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205163.944543:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205163.944545:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205163.944549:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944554:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205163.944556:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944558:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944560:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205163.944566:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944569:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205163.944570:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205163.944571:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205163.944573:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944574:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205163.944576:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944577:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944579:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944583:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205163.944584:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205163.944585:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205163.944587:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944588:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205163.944589:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944591:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205163.944593:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205163.944674:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205163.944685:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205163.944690:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205164.264525:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205164.264534:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1788205164.264536:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301291009, at 4301291520 00000100:00000040:1.0:1788205164.264536:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301291009, at 4301291520 00000100:00000001:3.0:1788205164.264538:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.264549:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205164.264554:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1788205164.264557:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000010:1.0:1788205164.264560:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00000001:3.0:1788205164.264560:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.264569:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.264572:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.264577:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.712801:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205164.712807:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.712809:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.712815:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205164.712818:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.712821:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.712825:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.712827:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.712830:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.776507:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205164.776514:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.776516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.776522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205164.776525:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.776529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.776532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.776535:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.776538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205164.776610:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205164.776616:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205164.776618:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205164.776621:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205164.776625:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205164.776631:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6619 00000400:00000200:2.0:1788205164.776639:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205164.776644:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205164.776648:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205164.776651:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776654:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205164.776656:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776659:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205164.776662:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205164.776664:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205164.776667:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205164.776671:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205164.776673:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205164.776675:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205164.776693:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776695:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205164.776697:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776699:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205164.776702:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205164.776705:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776708:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205164.776711:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205164.776714:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205164.776717:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.776719:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205164.776722:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205164.776730:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205164.776734:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205164.776738:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205164.776740:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.776743:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205164.776746:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205164.776749:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205164.776758:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205164.776764:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205164.776774:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205164.776783:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205164.776788:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205164.776792:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205164.776812:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205164.776815:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205164.776819:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205164.776820:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205164.776822:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205164.776825:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205164.776828:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6619 00000400:00000200:2.0:1788205164.776833:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205164.776837:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205164.776837:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788205164.776843:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205164.776843:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.776847:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205164.776848:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776851:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205164.776855:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205164.776856:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776861:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205164.776867:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205164.776869:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205164.776872:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788205164.776872:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.776876:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205164.776877:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 02000000:00000001:3.0:1788205164.776881:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788205164.776882:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788205164.776885:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788205164.776886:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:3.0:1788205164.776890:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205164.776893:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776897:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788205164.776899:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205935232:192.168.201.151@tcp:250:kworker.0 00000100:00000040:2.0:1788205164.776902:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.776907:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205164.776907:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1788205164.776912:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:3.0:1788205164.776913:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788205164.776916:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:3.0:1788205164.776917:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.776921:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.776923:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205164.776927:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1788205164.776930:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:2.0:1788205164.776932:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205164.776933:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205164.776937:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.776941:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205164.776944:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788205164.776946:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788205164.776949:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000008:00000001:2.0:1788205164.776952:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.776955:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205164.776957:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000010:2.0:1788205164.776960:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205164.776965:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000200:3.0:1788205164.776966:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935232, portal 25 02000000:00000001:2.0:1788205164.776968:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205164.776970:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788205164.776971:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.776974:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000001:2.0:1788205164.776975:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205164.776978:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205164.776981:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205164.776984:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205164.776987:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788205164.776993:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.776998:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788205164.777004:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205164.777004:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205164.777007:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205164.777013:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205935232, offset 0 00000100:00000001:2.0:1788205164.777019:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205164.777022:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.777024:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788205164.777029:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205164.777035:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205164.777044:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.777045:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:2.0:1788205164.777049:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205164.777053:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1788205164.777055:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205164.777058:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788205164.777059:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:2.0:1788205164.777061:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205164.777065:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788205164.777067:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205164.777071:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:2.0:1788205164.777075:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6619 00000400:00000200:3.0:1788205164.777080:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59162:59162:256:4294967295] 192.168.201.151@tcp LPNI seq info [59142:59142:8:4294967295] 00000400:00000200:3.0:1788205164.777085:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.777094:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205164.777100:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.777105:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205164.777111:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205164.777116:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205164.777123:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205164.777126:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205164.777132:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.777134:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205164.777140:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205164.777145:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205164.777151:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205164.777154:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205164.777158:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205164.777163:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777173:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205164.777179:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1788205164.777179:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205164.777183:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205164.777185:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205164.777188:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205164.777192:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205164.777195:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777196:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205164.777197:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205164.777200:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777201:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205164.777203:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777209:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.777215:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205164.777217:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301296129 00000100:00000010:3.0:1788205164.777222:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f40c0. 00000100:00000001:3.0:1788205164.777228:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205164.777234:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301296129 00000100:00000010:3.0:1788205164.777238:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4de0. 00000100:00000001:3.0:1788205164.777240:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205164.777243:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205164.777246:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777248:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205164.777253:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205164/real 1788205164] req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205164.777262:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205164.777263:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205164.777265:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205164.777268:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205164.777270:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205164.777272:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205164.777276:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205164.777283:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777290:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205164.777292:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777294:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.777296:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205164.777300:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777307:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205164.777312:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777318:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205164.777321:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777322:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.777323:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777326:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205164.777328:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205164.777331:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205164.777334:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205164.777337:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205164.777341:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205164.777379:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205935232:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205164.777384:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777386:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.777389:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205935232/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777396:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205164.777398:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205164.777401:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205164.777404:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205164.777406:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205164.777408:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205164.777410:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205164.777413:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:3.0:1788205164.777424:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205164.777429:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205164.777431:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:3.0:1788205164.777434:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205164.777436:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777438:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.777444:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.777445:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777447:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205164.777449:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.777450:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205164.777453:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777460:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.777461:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.777463:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205164.777465:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205164.777467:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.777470:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935360:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205164.777473:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205164.777475:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205164.777477:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.777478:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205164.777482:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205164.777484:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.777487:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205164.777490:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205164.777493:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205164.777496:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935360, portal 10 00000100:00000001:3.0:1788205164.777497:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205164.777498:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788205164.777505:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777511:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205164.777513:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205164.777515:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205164.777518:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205935360, offset 0 00000400:00000200:3.0:1788205164.777522:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205164.777610:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205164.777615:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205164.777618:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788205164.777620:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205164.777626:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:2.0:1788205164.777627:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777632:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205164.777632:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788205164.777638:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205164.777639:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:2.0:1788205164.777644:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205164.777645:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1788205164.777648:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:3.0:1788205164.777650:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:2.0:1788205164.777651:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205164.777655:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:3.0:1788205164.777656:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59163:59163:256:4294967295] 192.168.201.151@tcp LPNI seq info [59143:59143:8:4294967295] 00000020:00000040:2.0:1788205164.777658:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205164.777664:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:3.0:1788205164.777664:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:2.0:1788205164.777668:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205164.777674:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777689:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205164.777692:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000040:2.0:1788205164.777695:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777700:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:3.0:1788205164.777703:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000040:2.0:1788205164.777704:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000800:00000010:3.0:1788205164.777708:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00080000:2.0:1788205164.777711:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777718:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205164.777718:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788205164.777722:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1788205164.777724:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 02000000:00000001:2.0:1788205164.777728:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1788205164.777730:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788205164.777733:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205164.777735:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788205164.777736:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205164.777741:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000100:3.0:1788205164.777742:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.777746:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000008:00000001:2.0:1788205164.777747:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.777751:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.777752:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:2.0:1788205164.777756:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000400:00000200:3.0:1788205164.777760:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000040:2.0:1788205164.777762:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000200:3.0:1788205164.777765:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 02000000:00000001:2.0:1788205164.777767:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205164.777770:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205164.777770:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:2.0:1788205164.777775:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205164.777776:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788205164.777777:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205164.777781:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205164.777782:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205164.777786:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788205164.777793:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777795:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1788205164.777802:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205164.777803:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777829:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777833:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205164.777838:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.777839:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205164.777843:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205164.777843:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205164.777850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788205164.777853:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.777854:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301292187 00000100:00000001:3.0:1788205164.777859:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.777863:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301292187 00000100:00000001:3.0:1788205164.777867:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.777871:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205164.777874:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.777875:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.777877:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788205164.777881:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205164.777886:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205164/real 1788205164] req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205164.777887:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205164.777890:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205164.777893:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205164.777900:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:3.0:1788205164.777904:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788205164.777905:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6619 00000400:00000001:3.0:1788205164.777908:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205164.777911:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:2.0:1788205164.777915:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.777916:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205164.777920:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000400:00000200:2.0:1788205164.777923:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205164.777924:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1788205164.777929:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788205164.777931:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205164.777935:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777941:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205164.777945:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777950:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:3.0:1788205164.777950:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205164.777956:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205164.777959:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205164.777962:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205164.777967:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:3.0:1788205164.777967:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205164.777970:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205164.777972:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788205164.777974:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788205164.777976:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:3.0:1788205164.777977:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205164.777982:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:3.0:1788205164.777985:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205164.777986:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205164.777992:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205164.777997:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:3.0:1788205164.777999:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788205164.778003:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000040:3.0:1788205164.778007:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205164.778011:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205164.778017:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205164.778022:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205164.778023:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205164.778027:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205164.778027:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778031:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788205164.778033:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778035:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205164.778038:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205164.778039:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:2.0:1788205164.778042:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:3.0:1788205164.778043:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788205164.778048:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205164.778049:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788205164.778053:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205164.778056:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000010:2.0:1788205164.778060:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000100:00000001:3.0:1788205164.778060:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1788205164.778066:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000040:3.0:1788205164.778068:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205164.778071:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205164.778074:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.778079:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205164.778081:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205164.778085:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.778086:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935360:192.168.201.151@tcp:38:kworker.0 02000000:00000010:2.0:1788205164.778090:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:3.0:1788205164.778092:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205164.778096:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778096:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.778102:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205935360/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205164.778105:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205164.778123:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205164.778125:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788205164.778127:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:3.0:1788205164.778129:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 00000100:00000001:2.0:1788205164.778131:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.778133:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205164.778136:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:2.0:1788205164.778138:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205164.778139:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205164.778144:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205164.778146:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:3.0:1788205164.778151:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205164.778153:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788205164.778155:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205164.778156:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:3.0:1788205164.778161:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1788205164.778163:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6629) 00000100:00000001:3.0:1788205164.778163:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778168:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778172:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.778173:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778176:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205164.778178:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.778179:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205164.778184:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778191:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.778193:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.778194:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205164.778196:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205164.778198:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.778201:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935488:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205164.778204:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205164.778206:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205164.778207:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.778209:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205164.778212:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205164.778214:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.778217:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205164.778220:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205164.778223:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205164.778225:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935488, portal 4 00000100:00000001:3.0:1788205164.778227:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205164.778228:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788205164.778233:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778239:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205164.778242:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205164.778244:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205164.778246:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205935488, offset 0 00000400:00000200:3.0:1788205164.778249:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205164.778253:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.778257:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205164.778260:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205164.778263:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205164.778265:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205164.778268:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205164.778271:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59164:59164:256:4294967295] 192.168.201.151@tcp LPNI seq info [59144:59144:8:4294967295] 00000400:00000200:3.0:1788205164.778274:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.778280:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205164.778285:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.778288:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205164.778291:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205164.778294:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205164.778298:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205164.778301:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205164.778304:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.778306:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205164.778309:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205164.778312:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205164.778315:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205164.778317:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205164.778319:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205164.778322:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778330:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.778333:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778340:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778342:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205164.778345:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205164.778347:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205164.778350:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205164.778352:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301292073 00000100:00000001:3.0:1788205164.778355:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.778357:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301292073 00000100:00000001:3.0:1788205164.778359:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.778377:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205164.778380:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778381:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205164.778385:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205164/real 1788205164] req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205164.778394:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205164.778395:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205164.778397:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205164.778399:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205164.778401:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205164.778403:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205164.778406:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205164.778414:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778421:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205164.778422:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778423:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.778424:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205164.778428:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778435:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205164.778439:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778445:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205164.778447:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778449:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.778450:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778452:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205164.778454:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205164.778457:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205164.778460:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205164.778462:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205164.778466:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205164.778474:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935488:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205164.778478:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205164.778481:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778487:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.778489:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.778491:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205164.778492:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205164.778494:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.778496:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205164.778499:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205164.778501:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205164.778502:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205164.778504:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205164.778507:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205164.778509:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205164.778512:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205164.778514:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205164.778517:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205164.778519:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935616, portal 4 00000100:00000001:3.0:1788205164.778521:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205164.778523:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788205164.778527:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778534:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205164.778536:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205164.778538:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205164.778541:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205935616, offset 0 00000400:00000200:3.0:1788205164.778544:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205164.778548:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.778551:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205164.778554:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205164.778556:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205164.778558:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205164.778561:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205164.778563:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59165:59165:256:4294967295] 192.168.201.151@tcp LPNI seq info [59145:59145:8:4294967295] 00000400:00000200:3.0:1788205164.778567:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205164.778573:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205164.778578:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.778581:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205164.778584:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205164.778587:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205164.778591:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205164.778593:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205164.778597:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205164.778599:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205164.778601:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205164.778605:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205164.778607:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205164.778609:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205164.778611:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205164.778615:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778622:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.778625:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778631:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778633:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205164.778635:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205164.778638:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205164.778641:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205164.778643:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301292080 00000100:00000001:3.0:1788205164.778645:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.778647:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301292080 00000100:00000001:3.0:1788205164.778649:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301296129 00000100:00000001:3.0:1788205164.778651:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205164.778653:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778654:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205164.778658:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205164/real 1788205164] req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205164.778665:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205164.778666:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205164.778667:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205164.778670:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205164.778671:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205164.778673:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205164.778676:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205164.778697:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778705:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205164.778706:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778708:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.778709:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205164.778713:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778719:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205164.778723:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778730:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205164.778732:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778734:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205164.778735:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778737:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205164.778738:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205164.778741:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205164.778744:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205164.778746:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205164.778749:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205164.778757:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935616:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205164.778761:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778763:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.778766:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205935488/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778773:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205164.778774:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205164.778776:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205164.778778:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205164.778779:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205164.778781:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205164.778783:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205164.778785:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:3.0:1788205164.778787:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205164.778788:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205164.778789:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:3.0:1788205164.778792:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205164.778793:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778795:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205164.778798:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205935616/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205164.778803:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205164.778804:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205164.778806:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205164.778807:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205164.778808:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205164.778810:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205164.778812:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205164.778813:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:3.0:1788205164.778815:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205164.778816:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205164.778817:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788205164.778819:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205164.778820:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778821:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.778825:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.778826:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205164.778828:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205164.778829:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205164.968482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205164.968489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205164.968495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205164.968497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205164.968499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205164.968501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205164.968504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205164.968505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205164.968561:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205164.968564:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205164.968566:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205165.288586:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205165.288593:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205165.288595:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205165.288600:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205165.288603:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.288607:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.288610:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205165.288612:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.288616:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205165.736511:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205165.736518:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205165.736519:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205165.736525:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205165.736528:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.736532:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.736534:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205165.736537:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.736539:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205165.800427:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205165.800436:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205165.800438:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205165.800444:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205165.800448:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.800452:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.800455:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205165.800458:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.800466:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205165.800584:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205165.800587:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205165.800589:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205165.800591:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205165.800593:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.800596:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.800599:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205165.800600:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205165.800603:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205165.992529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205165.992538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205165.992544:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205165.992545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205165.992549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205165.992550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205165.992553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205165.992555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205165.992634:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205165.992639:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205165.992643:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205166.313814:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205166.313818:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205166.313820:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205166.313825:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205166.313828:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205166.313832:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205166.313835:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205166.313837:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205166.313840:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.760520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205166.760530:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205166.760533:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.760540:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205166.760545:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205166.760549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.760552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.760555:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.760559:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205166.824532:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205166.824533:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205166.824541:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205166.824541:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205166.824544:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205166.824545:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205166.824553:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205166.824553:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205166.824559:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205166.824560:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205166.824564:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.824567:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205166.824569:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.824570:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205166.824577:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.824578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205166.824581:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205166.824586:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205167.016456:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205167.016465:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205167.016472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205167.016474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205167.016479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205167.016481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205167.016485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205167.016486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205167.016548:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205167.016552:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205167.016556:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205167.336462:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205167.336471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205167.336473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205167.336480:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205167.336485:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205167.336490:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205167.336493:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205167.336497:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205167.336501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205167.785396:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205167.785404:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205167.785406:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205167.785423:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205167.785426:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205167.785430:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.785432:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205167.785435:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.785438:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205167.848510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205167.848510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205167.848520:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205167.848520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205167.848524:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205167.848525:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205167.848534:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205167.848534:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205167.848541:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205167.848541:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205167.848548:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.848549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205167.848553:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205167.848558:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205167.848562:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205167.848562:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205167.848565:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.848570:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205168.040529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205168.040535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205168.040538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205168.040539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205168.040541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205168.040541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205168.040543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205168.040544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205168.040603:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205168.040609:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205168.040613:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205168.360524:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205168.360534:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205168.360537:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205168.360544:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205168.360549:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205168.360553:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205168.360557:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205168.360560:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205168.360563:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205168.808525:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205168.808535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205168.808538:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205168.808545:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205168.808549:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205168.808553:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.808556:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205168.808559:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.808563:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205168.872503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205168.872503:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205168.872513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205168.872516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205168.872517:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205168.872519:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205168.872527:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205168.872527:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205168.872532:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205168.872532:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205168.872539:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.872539:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205168.872545:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205168.872550:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.872554:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205168.872555:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205168.872558:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.872562:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205169.064508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205169.064518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205169.064527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205169.064528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205169.064532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205169.064534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205169.064538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205169.064539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205169.064623:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205169.064627:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205169.064631:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205169.384514:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301296129, at 4301296640 00000100:00000010:1.0:1788205169.384536:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4de0. 00000100:00000040:0.0:1788205169.386041:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301296129, at 4301296641 00000100:00000010:0.0:1788205169.386050:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f40c0. 00000100:00000001:3.0:1788205169.386053:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205169.386060:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.386063:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.386068:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205169.386072:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205169.386077:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.386080:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.386083:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.386087:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.832482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205169.832492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.832494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.832502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205169.832506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205169.832510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.832513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.832516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.832519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205169.896519:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205169.896519:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205169.896527:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.896530:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205169.896532:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1788205169.896536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205169.896537:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205169.896542:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205169.896542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205169.896543:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205169.896546:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205169.896548:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205169.896550:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205169.896550:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205169.896552:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.896554:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205169.896557:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205169.896557:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205169.896563:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205169.896565:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205169.896568:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205169.896570:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205169.896572:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205169.896723:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6624 00000400:00000200:2.0:1788205169.896735:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205169.896742:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205169.896745:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205169.896749:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896752:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205169.896755:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896757:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205169.896767:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205169.896770:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205169.896773:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205169.896776:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205169.896779:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205169.896782:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205169.896788:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896790:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205169.896792:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896794:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205169.896797:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205169.896800:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896803:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205169.896806:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205169.896810:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205169.896813:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.896816:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205169.896819:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205169.896830:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205169.896834:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205169.896838:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205169.896840:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.896843:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205169.896846:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205169.896851:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205169.896860:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788205169.896868:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205169.896879:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205169.896891:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205169.896896:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205169.896902:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205169.896920:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205169.896922:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205169.896924:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205169.896926:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205169.896927:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205169.896929:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205169.896931:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6624 00000400:00000200:2.0:1788205169.896934:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205169.896936:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205169.896938:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205169.896940:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896942:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205169.896944:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896945:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205169.896947:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205169.896948:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205169.896949:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205169.896951:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205169.896952:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205169.896953:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788205169.896953:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788205169.896958:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896960:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205169.896960:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205169.896964:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205169.896964:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205169.896966:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205169.896969:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205169.896972:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205169.896975:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205169.896976:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1788205169.896976:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205169.896982:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205169.896986:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.896989:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205169.896990:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.896994:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205169.896995:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205169.896999:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897000:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.897001:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1788205169.897003:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000001:3.0:1788205169.897005:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1788205169.897006:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205169.897011:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205169.897012:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205169.897015:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897017:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788205169.897019:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205935744:192.168.201.151@tcp:250:kworker.0 02000000:00000001:2.0:1788205169.897022:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205169.897026:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897028:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:2.0:1788205169.897032:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788205169.897034:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788205169.897037:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.897039:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.897044:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788205169.897045:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1788205169.897052:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 00000100:00000001:2.0:1788205169.897054:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205169.897056:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897059:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205169.897067:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205169.897067:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205169.897073:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:2.0:1788205169.897082:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205169.897083:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00080000:2.0:1788205169.897086:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205169.897090:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205169.897093:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1788205169.897093:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935744, portal 25 00000100:00080000:2.0:1788205169.897097:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205169.897098:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205169.897102:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000001:2.0:1788205169.897104:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205169.897108:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6624 00000100:00000040:3.0:1788205169.897112:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205169.897117:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205169.897124:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205169.897128:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788205169.897129:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205169.897135:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:3.0:1788205169.897136:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897137:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1788205169.897140:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000040:2.0:1788205169.897141:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897144:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:3.0:1788205169.897148:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205935744, offset 0 00000020:00000001:2.0:1788205169.897152:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205169.897154:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205169.897156:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205169.897159:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:3.0:1788205169.897160:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788205169.897161:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205169.897164:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205169.897167:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897169:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205169.897170:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897175:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:3.0:1788205169.897175:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788205169.897180:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205169.897182:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897184:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205169.897186:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205169.897189:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205169.897191:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897193:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205169.897196:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:3.0:1788205169.897197:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00000001:2.0:1788205169.897200:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897203:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205169.897205:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:2.0:1788205169.897209:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000400:00000200:3.0:1788205169.897213:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000040:2.0:1788205169.897214:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205169.897219:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788205169.897220:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1788205169.897221:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897226:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205169.897228:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205169.897230:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205169.897233:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000400:00000200:3.0:1788205169.897235:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897237:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205169.897242:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59166:59166:256:4294967295] 192.168.201.151@tcp LPNI seq info [59146:59146:8:4294967295] 00000100:00100000:2.0:1788205169.897245:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205169.897254:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205169.897256:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205169.897259:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205169.897262:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205169.897273:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205169.897279:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205169.897284:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1788205169.897287:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205169.897289:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205169.897292:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205169.897294:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:3.0:1788205169.897296:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000100:00000001:2.0:1788205169.897298:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205169.897303:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6624 00000800:00000200:3.0:1788205169.897307:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000400:00000200:2.0:1788205169.897309:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205169.897315:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000200:3.0:1788205169.897316:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00080000:2.0:1788205169.897317:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205169.897319:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897321:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205169.897322:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897323:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205169.897324:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205169.897326:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205169.897329:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205169.897333:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000200:3.1:1788205169.897337:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788205169.897338:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205169.897345:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:3.1:1788205169.897345:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1788205169.897352:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000100:3.0:1788205169.897357:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205169.897358:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1788205169.897386:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000100:00000040:2.0:1788205169.897392:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:3.0:1788205169.897395:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205169.897397:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205169.897402:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000200:3.0:1788205169.897403:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205169.897410:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000100:00080000:2.0:1788205169.897411:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000010:3.0:1788205169.897414:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:2.0:1788205169.897416:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897419:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788205169.897420:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205169.897425:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1788205169.897426:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205169.897431:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897435:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205169.897437:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205169.897441:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897442:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897444:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1788205169.897447:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205169.897451:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000040:3.0:1788205169.897451:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205169.897455:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205169.897457:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897460:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205169.897462:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205169.897464:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205169.897470:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:3.0:1788205169.897470:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897474:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205169.897475:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205169.897482:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205169.897482:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.897492:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205169.897492:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205169.897496:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897498:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301301250 00000100:00000040:2.0:1788205169.897501:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205169.897508:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205169.897509:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f48a0. 00000100:00000040:2.0:1788205169.897512:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6634) 00000100:00000001:3.0:1788205169.897519:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205169.897528:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301301250 00000100:00000010:3.0:1788205169.897533:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4120. 00000100:00000001:3.0:1788205169.897537:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205169.897540:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205169.897543:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897545:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205169.897551:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205169/real 1788205169] req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205169.897566:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205169.897568:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205169.897570:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205169.897572:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205169.897574:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205169.897576:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205169.897579:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205169.897588:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897611:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205169.897612:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897615:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.897617:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205169.897621:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897628:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205169.897632:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897639:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205169.897642:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897644:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.897645:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897648:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205169.897650:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205169.897653:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205169.897656:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205169.897659:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205169.897663:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205169.897671:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205935744:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205169.897676:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897678:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.897681:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205935744/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897687:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205169.897689:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205169.897692:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205169.897696:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205169.897698:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205169.897700:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:3.0:1788205169.897702:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205169.897704:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:3.0:1788205169.897710:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205169.897712:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205169.897714:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:3.0:1788205169.897717:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205169.897718:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897720:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897729:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.897730:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897732:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205169.897734:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205169.897735:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205169.897745:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897752:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.897753:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.897756:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205169.897758:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205169.897760:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.897763:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935872:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205169.897767:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205169.897769:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205169.897771:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.897772:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205169.897775:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205169.897777:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205169.897781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205169.897783:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205169.897787:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205169.897790:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205935872, portal 10 00000100:00000001:3.0:1788205169.897793:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205169.897794:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:3.0:1788205169.897801:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897807:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205169.897817:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205169.897820:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205169.897824:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205935872, offset 0 00000400:00000200:3.0:1788205169.897828:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205169.897832:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.897836:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205169.897839:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205169.897842:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205169.897844:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205169.897847:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205169.897849:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59167:59167:256:4294967295] 192.168.201.151@tcp LPNI seq info [59147:59147:8:4294967295] 00000400:00000200:3.0:1788205169.897853:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.897859:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205169.897864:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.897867:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205169.897870:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205169.897874:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205169.897878:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205169.897881:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205169.897885:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.897887:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205169.897890:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205169.897893:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205169.897896:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205169.897898:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205169.897901:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205169.897904:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897912:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.897915:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897921:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897922:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205169.897925:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205169.897929:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205169.897932:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205169.897934:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301297307 00000100:00000001:3.0:1788205169.897937:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.897939:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301297307 00000100:00000001:3.0:1788205169.897941:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.897943:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205169.897945:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.897947:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205169.897951:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205169/real 1788205169] req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205169.897959:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205169.897960:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205169.897962:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205169.897965:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205169.897966:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205169.897968:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205169.897971:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205169.897980:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.897986:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205169.897987:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.897989:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.897990:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205169.897994:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898000:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205169.898004:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898010:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205169.898013:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898014:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.898015:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898018:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205169.898020:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205169.898023:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205169.898026:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205169.898028:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205169.898032:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205169.898040:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205935872:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205169.898044:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205169.898047:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898054:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.898055:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.898057:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205169.898060:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205169.898062:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.898065:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936000:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205169.898068:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205169.898070:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205169.898072:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.898073:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205169.898077:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205169.898079:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205169.898082:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205169.898085:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205169.898088:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205169.898090:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936000, portal 4 00000100:00000001:3.0:1788205169.898093:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205169.898094:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:3.0:1788205169.898100:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898106:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205169.898109:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205169.898111:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205169.898113:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205936000, offset 0 00000400:00000200:3.0:1788205169.898117:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205169.898121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.898125:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205169.898127:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205169.898130:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205169.898133:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205169.898135:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205169.898138:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59168:59168:256:4294967295] 192.168.201.151@tcp LPNI seq info [59148:59148:8:4294967295] 00000400:00000200:3.0:1788205169.898142:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.898149:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205169.898154:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.898157:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205169.898161:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205169.898164:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205169.898168:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205169.898171:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205169.898175:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.898177:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205169.898179:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205169.898183:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205169.898186:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205169.898187:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205169.898190:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205169.898193:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898200:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.898203:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898208:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.898210:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205169.898213:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205169.898216:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:3.0:1788205169.898219:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205169.898221:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301297193 00000100:00000001:3.0:1788205169.898223:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.898226:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301297193 00000100:00000001:3.0:1788205169.898228:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.898230:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205169.898232:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.898234:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205169.898238:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205169/real 1788205169] req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205169.898245:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205169.898246:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205169.898248:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205169.898250:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205169.898252:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205169.898254:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205169.898257:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205169.898266:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898273:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205169.898274:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898277:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.898278:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205169.898281:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898288:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205169.898292:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898299:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205169.898301:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898303:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.898304:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898307:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205169.898309:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205169.898312:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205169.898315:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205169.898317:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205169.898321:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205169.898329:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936000:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205169.898333:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205169.898336:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898342:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.898343:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.898345:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205169.898347:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205169.898349:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.898352:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936128:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205169.898355:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205169.898357:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205169.898358:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205169.898360:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205169.898384:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205169.898386:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205169.898389:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205169.898392:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000400:00000010:3.0:1788205169.898395:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000200:3.0:1788205169.898399:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936128, portal 4 00000100:00000001:3.0:1788205169.898401:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205169.898402:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:3.0:1788205169.898408:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898415:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205169.898418:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205169.898420:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205169.898423:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205936128, offset 0 00000400:00000200:3.0:1788205169.898426:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205169.898430:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.898433:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205169.898436:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205169.898438:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205169.898440:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205169.898443:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205169.898446:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59169:59169:256:4294967295] 192.168.201.151@tcp LPNI seq info [59149:59149:8:4294967295] 00000400:00000200:3.0:1788205169.898450:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205169.898455:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205169.898460:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.898463:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205169.898468:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.0:1788205169.898470:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205169.898474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205169.898477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205169.898481:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205169.898483:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:3.0:1788205169.898486:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205169.898489:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205169.898492:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:3.0:1788205169.898494:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:3.0:1788205169.898496:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205169.898501:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898508:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.898512:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898524:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.898525:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205169.898528:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205169.898531:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:3.0:1788205169.898535:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205169.898536:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301297200 00000100:00000001:3.0:1788205169.898539:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.898541:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301297200 00000100:00000001:3.0:1788205169.898543:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301301250 00000100:00000001:3.0:1788205169.898546:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205169.898548:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.898549:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205169.898553:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205169/real 1788205169] req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205169.898561:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205169.898563:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205169.898564:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:3.0:1788205169.898566:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000400:00000010:3.0:1788205169.898569:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000001:3.0:1788205169.898571:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205169.898574:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205169.898583:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898602:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205169.898604:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898606:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.898608:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205169.898613:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898620:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205169.898624:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898630:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205169.898632:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898634:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205169.898635:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898637:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205169.898639:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205169.898641:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205169.898644:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:3.0:1788205169.898646:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205169.898650:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205169.898658:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936128:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205169.898662:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898663:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.898666:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205935872/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898673:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205169.898674:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205169.898676:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205169.898679:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205169.898681:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205169.898682:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:3.0:1788205169.898684:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205169.898686:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:3.0:1788205169.898689:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205169.898690:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205169.898692:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:3.0:1788205169.898695:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205169.898696:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898697:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.898701:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205936000/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898707:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205169.898708:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205169.898710:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205169.898712:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205169.898713:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205169.898715:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:3.0:1788205169.898717:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205169.898718:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:3.0:1788205169.898720:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205169.898721:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205169.898722:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:3.0:1788205169.898724:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205169.898726:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898727:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205169.898731:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205936128/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205169.898737:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205169.898738:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205169.898740:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205169.898741:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205169.898743:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1788205169.898744:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:3.0:1788205169.898746:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1788205169.898748:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:3.0:1788205169.898750:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205169.898751:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205169.898752:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:3.0:1788205169.898754:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205169.898756:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898758:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.898762:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.898764:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.898766:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205169.898767:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205170.088686:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205170.088697:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205170.088704:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205170.088706:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205170.088709:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205170.088711:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205170.088714:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205170.088715:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205170.088866:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205170.088870:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205170.088874:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205170.408515:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205170.408524:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205170.408525:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205170.408530:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205170.408534:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205170.408537:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.408541:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205170.408544:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.408547:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205170.856489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205170.856506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205170.856508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205170.856513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205170.856516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205170.856520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.856522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205170.856524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.856527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205170.920492:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205170.920501:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205170.920502:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205170.920509:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205170.920510:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205170.920512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205170.920522:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205170.920523:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205170.920530:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205170.920531:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205170.920536:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.920538:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205170.920540:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205170.920544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205170.920551:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205170.920552:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205170.920556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.920560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205171.112497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112503:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205171.112506:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205171.112509:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205171.112513:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112516:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205171.112518:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112519:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112521:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205171.112525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205171.112528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205171.112529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205171.112609:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205171.112614:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205171.112617:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205171.432517:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205171.432528:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.432530:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205171.432536:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205171.432539:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.432544:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.432548:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.432550:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.432553:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.880531:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205171.880543:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.880546:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205171.880554:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205171.880571:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.880576:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.880579:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.880583:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.880587:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.944538:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205171.944550:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.944551:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205171.944573:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205171.944584:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.944589:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.944593:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.944597:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.944601:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.945417:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205171.945423:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205171.945425:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205171.945429:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205171.945432:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205171.945434:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205171.945436:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205171.945438:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205171.945440:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205172.136538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205172.136546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205172.136603:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205172.136605:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205172.136608:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205172.136609:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205172.136613:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205172.136614:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205172.137542:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205172.137566:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205172.137572:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205172.456454:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205172.456461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205172.456463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205172.456470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205172.456473:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205172.456477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.456480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205172.456483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.456486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205172.904518:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205172.904530:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205172.904532:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205172.904557:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205172.904563:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205172.904567:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.904570:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205172.904574:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.904577:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205172.968521:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205172.968521:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205172.968535:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205172.968548:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205172.968552:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205172.968553:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205172.968564:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205172.968568:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205172.968571:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205172.968573:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.968581:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205172.968582:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205172.968585:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205172.968588:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.968589:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205172.968598:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205172.968602:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205172.968605:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205173.160512:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205173.160516:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205173.160520:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160527:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205173.160529:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160531:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160542:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205173.160548:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205173.160552:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205173.160553:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205173.160555:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160556:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205173.160557:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160558:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160559:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160561:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205173.160562:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205173.160563:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205173.160564:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160565:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205173.160565:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160566:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205173.160568:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205173.160669:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205173.160675:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205173.160680:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205173.480511:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205173.480522:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205173.480526:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205173.480542:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205173.480547:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205173.480556:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.480559:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205173.480562:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.480566:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205173.928503:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205173.928510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205173.928513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205173.928518:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205173.928532:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205173.928536:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.928538:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205173.928541:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.928544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205173.992469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205173.992475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205173.992477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205173.992482:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205173.992485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205173.992488:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205173.992490:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205173.992492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205173.992494:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205173.992502:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205173.992506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205173.992507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205173.992510:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205173.992513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205173.992515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.992517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205173.992529:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205173.992532:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205174.184573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205174.184582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205174.184587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205174.184589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205174.184592:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205174.184594:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205174.184597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205174.184598:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205174.184930:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205174.184936:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205174.184940:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205174.504477:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205174.504488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205174.504490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205174.504495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205174.504498:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205174.504502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.504536:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205174.504541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.504545:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205174.504634:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301301250, at 4301301760 00000100:00000010:0.0:1788205174.504642:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f48a0. 00000100:00000040:1.0:1788205174.505600:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301301250, at 4301301761 00000100:00000010:1.0:1788205174.505616:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4120. 00000100:00000001:3.0:1788205174.952553:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205174.952561:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205174.952563:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205174.952569:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205174.952572:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205174.952576:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.952578:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205174.952580:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.952582:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205175.016471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205175.016476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205175.016477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205175.016481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205175.016484:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205175.016487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205175.016489:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205175.016492:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205175.016495:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205175.016528:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205175.016534:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205175.016536:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205175.016539:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205175.016543:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205175.016549:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6629 00000400:00000200:2.0:1788205175.016557:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205175.016563:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205175.016566:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205175.016568:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.016571:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.016573:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.016576:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205175.016579:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205175.016581:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205175.016584:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205175.016587:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205175.016589:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205175.016591:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205175.016593:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.016594:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.016597:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.016599:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205175.016601:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205175.016604:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205175.016607:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205175.016609:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205175.016613:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205175.016615:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.016617:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205175.016619:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205175.016628:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205175.016632:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205175.016635:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205175.016637:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.016639:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205175.016642:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205175.016645:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205175.016654:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788205175.016661:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205175.016669:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205175.016678:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205175.016682:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205175.016687:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205175.016707:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205175.017206:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205175.017210:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205175.017211:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205175.017213:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205175.017216:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205175.017220:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6629 00000400:00000200:2.0:1788205175.017224:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205175.017228:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205175.017230:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205175.017233:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017235:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017237:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017239:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205175.017242:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205175.017243:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205175.017245:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205175.017247:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205175.017249:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205175.017251:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205175.017254:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017256:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017258:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017260:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205175.017262:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205175.017265:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017267:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205175.017269:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205175.017273:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205175.017276:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017277:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205175.017282:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205175.017284:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017286:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205175.017290:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205175.017292:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205175.017295:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205175.017296:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017298:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205175.017299:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205175.017302:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205175.017306:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788205175.017310:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205175.017315:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205175.017322:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205175.017324:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205175.017328:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205175.017335:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205175.017338:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205175.017341:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205175.017342:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205175.017344:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205175.017346:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205175.017348:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6629 00000400:00000200:2.0:1788205175.017351:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205175.017354:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205175.017356:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205175.017359:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017361:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017384:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017387:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205175.017389:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205175.017390:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205175.017391:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205175.017394:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205175.017395:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205175.017398:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205175.017401:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017402:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017404:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017406:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205175.017408:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205175.017410:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017412:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205175.017414:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205175.017417:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205175.017419:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017420:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205175.017422:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205175.017425:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017426:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205175.017430:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205175.017432:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205175.017434:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205175.017435:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017437:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205175.017438:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205175.017440:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205175.017443:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205175.017446:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205175.017451:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205175.017469:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205175.017471:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205175.017475:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205175.017482:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205175.017485:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205175.017488:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205175.017489:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205175.017491:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205175.017493:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205175.017495:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6629 00000400:00000200:2.0:1788205175.017499:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205175.017514:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205175.017516:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205175.017518:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017520:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017522:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017524:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205175.017526:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205175.017527:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205175.017528:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205175.017531:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205175.017533:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205175.017535:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205175.017537:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017539:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205175.017541:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017542:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205175.017544:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205175.017547:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205175.017549:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205175.017551:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205175.017554:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205175.017556:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017558:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205175.017560:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205175.017563:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017564:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205175.017568:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205175.017570:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205175.017573:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205175.017574:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.017575:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205175.017577:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205175.017578:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205175.017582:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205175.017585:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205175.017590:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205175.017597:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205175.017599:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205175.017602:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205175.017610:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205175.017614:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6639) 00000100:00000001:3.0:1788205175.018934:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205175.018939:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205175.018942:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205175.018951:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.018961:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205175.018964:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.018968:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205175.018971:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205175.018974:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205175.018978:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205936256:192.168.201.151@tcp:250:kworker.0 00000100:00000001:3.0:1788205175.018982:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205175.018986:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205175.018988:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.018991:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788205175.018995:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 02000000:00000010:3.0:1788205175.018998:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21c00. 02000000:00000001:3.0:1788205175.019004:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.019006:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205175.019010:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205175.019013:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:1.0:1788205175.019018:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000400:00000010:3.0:1788205175.019020:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5c60. 00000100:00000001:1.0:1788205175.019024:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000010:3.0:1788205175.019030:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f43b8. 00000100:00000001:1.0:1788205175.019031:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205175.019037:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000200:3.0:1788205175.019039:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936256, portal 25 00000100:00000001:1.0:1788205175.019040:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205175.019048:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205175.019048:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205175.019055:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:3.0:1788205175.019065:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.019072:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205175.019075:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205175.019078:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4330. 00000100:00000200:3.0:1788205175.019094:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205936256, offset 0 00000400:00000200:3.0:1788205175.019101:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205175.019109:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205175.019116:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205175.019121:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205175.019125:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205175.019129:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205175.019131:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205175.019134:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59170:59170:256:4294967295] 192.168.201.151@tcp LPNI seq info [59150:59150:8:4294967295] 00000400:00000200:3.0:1788205175.019139:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205175.019148:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205175.019154:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205175.019158:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ec800. 00000800:00000200:3.0:1788205175.019165:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205175.019169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788205175.019192:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205175.019196:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205175.019199:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301306371 00000100:00000010:3.0:1788205175.019203:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f4000. 00000100:00000001:3.0:1788205175.019209:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788205175.019216:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301306371 00000100:00000010:3.0:1788205175.019218:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3104f41e0. 00000100:00000001:3.0:1788205175.019221:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788205175.019225:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205175.019227:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205175.019230:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205175.019235:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.019242:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205175.019244:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.019247:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205175.019248:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205175.019250:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205175.019253:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936384:192.168.201.151@tcp:38:kworker.0 00000100:00000001:3.0:1788205175.019256:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205175.019258:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205175.019260:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.019261:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205175.019264:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c21000. 02000000:00000001:3.0:1788205175.019266:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205175.019269:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205175.019272:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5f20. 00000400:00000010:3.0:1788205175.019275:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4088. 00000100:00000200:3.0:1788205175.019278:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936384, portal 10 00000100:00000001:3.0:1788205175.019280:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205175.019282:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:3.0:1788205175.019288:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.019294:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205175.019295:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:3.0:1788205175.019297:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4e58. 00000100:00000200:3.0:1788205175.019298:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205936384, offset 0 00000400:00000200:3.0:1788205175.019301:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:3.0:1788205175.019304:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205175.019307:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:3.0:1788205175.019309:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205175.019311:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:3.0:1788205175.019312:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205175.019314:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:3.0:1788205175.019316:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59171:59171:255:4294967295] 192.168.201.151@tcp LPNI seq info [59151:59151:7:4294967295] 00000400:00000200:3.0:1788205175.019318:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205175.019322:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:3.0:1788205175.019325:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:3.0:1788205175.019327:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c30a0ecb00. 00000800:00000200:3.0:1788205175.019330:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:3.1:1788205175.019333:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:3.0:1788205175.019336:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205175.019339:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:3.0:1788205175.019342:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205175.019344:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301302428 00000100:00000001:3.0:1788205175.019346:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301306371 00000100:00000001:3.0:1788205175.019348:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301302428 00000100:00000001:3.0:1788205175.019350:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301306371 00000100:00000001:3.0:1788205175.019352:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205175.019353:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205175.019356:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205175.019360:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.019394:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205175.019395:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.019396:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205175.019398:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205175.019399:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205175.019401:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936512:192.168.201.151@tcp:8:kworker.0 00000100:00000001:3.0:1788205175.019406:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205175.019408:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205175.019410:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205175.019411:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205175.019414:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310c22c00. 02000000:00000001:3.0:1788205175.019415:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205175.019417:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:3.0:1788205175.019419:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3066f5a50. 00000400:00000010:3.0:1788205175.019425:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3066f4aa0. 00000100:00000200:3.0:1788205175.019428:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936512, portal 4 00000100:00000001:3.0:1788205175.019432:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205175.019434:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:3.0:1788205175.019449:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205175.019455:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205175.019457:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205175.019525:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205175.019535:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205936512, offset 0 00000400:00000200:1.0:1788205175.019545:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205175.019553:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205175.019559:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205175.019563:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205175.019566:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205175.019569:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205175.019571:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205175.019574:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59172:59172:254:4294967295] 192.168.201.151@tcp LPNI seq info [59152:59152:6:4294967295] 00000400:00000200:1.0:1788205175.019577:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205175.019585:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205175.019590:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205175.019595:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264300. 00000800:00000200:1.0:1788205175.019600:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1F:1788205175.019604:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205175.019620:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205175.019625:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205175.019632:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205175.019634:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301302315 00000100:00000001:1.0:1788205175.019637:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301306371 00000100:00000001:1.0:1788205175.019640:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301302315 00000100:00000001:1.0:1788205175.019642:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301306371 00000100:00000001:1.0:1788205175.019645:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205175.019648:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.019651:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205175.019659:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205175.019676:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205175.019678:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205175.019682:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205175.019685:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205175.019688:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205175.019692:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936640:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205175.019696:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205175.019700:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205175.019702:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205175.019704:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205175.019710:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205175.019713:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205175.019718:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205175.019723:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b1b8. 00000400:00000010:1.0:1788205175.019729:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657440. 00000100:00000200:1.0:1788205175.019732:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936640, portal 4 00000100:00000001:1.0:1788205175.019734:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205175.019735:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:1.0:1788205175.019741:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205175.019746:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205175.019748:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205175.019751:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205175.019753:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205936640, offset 0 00000400:00000200:1.0:1788205175.019757:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205175.019761:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205175.019764:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205175.019767:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205175.019769:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205175.019771:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205175.019774:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205175.019777:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59173:59173:253:4294967295] 192.168.201.151@tcp LPNI seq info [59153:59153:5:4294967295] 00000400:00000200:1.0:1788205175.019781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205175.019791:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205175.019799:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205175.019802:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205175.019805:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205175.019809:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205175.019810:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205175.019813:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205175.019815:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205175.019816:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301302321 00000100:00000001:1.0:1788205175.019818:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301306371 00000100:00000001:1.0:1788205175.019819:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301302321 00000100:00000001:1.0:1788205175.019821:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301306371 00000100:00000001:1.0:1788205175.019822:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205175.019824:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.019827:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.019830:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205175.021520:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205175.021528:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788205175.021535:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205175.021538:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205175.021543:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.021546:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205175.208522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205175.208532:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205175.208540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205175.208542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205175.208545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205175.208547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205175.208550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205175.208552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205175.208775:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205175.208779:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205175.208783:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205175.532685:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205175.532694:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205175.532697:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205175.532703:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205175.532707:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205175.532712:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205175.532715:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205175.532718:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205175.532721:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205175.976495:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205175.976502:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205175.976504:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205175.976510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205175.976513:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205175.976516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205175.976519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205175.976521:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205175.976524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.040520:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205176.040528:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205176.040530:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.040536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205176.040540:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205176.040544:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.040546:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.040549:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.040554:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205176.040614:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205176.040621:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205176.040623:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205176.040628:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205176.040631:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205176.040634:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205176.040637:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205176.040639:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205176.040642:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205176.232529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205176.232536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205176.232541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205176.232542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205176.232545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205176.232547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205176.232549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205176.232550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205176.232674:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205176.232678:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205176.232680:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205176.552465:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205176.552471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205176.552472:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205176.552506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205176.552510:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205176.552514:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205176.552516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205176.552519:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205176.552522:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205177.000462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205177.000481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205177.000483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205177.000488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205177.000498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205177.000502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205177.000505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205177.000507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205177.000510:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205177.064558:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205177.064566:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205177.064568:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205177.064573:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205177.064577:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205177.064581:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205177.064584:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205177.064587:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205177.064590:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205177.064600:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205177.064604:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205177.064606:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205177.064609:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205177.064612:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205177.064615:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205177.064617:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205177.064618:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205177.064621:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205177.256494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205177.256500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205177.256506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205177.256508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205177.256511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205177.256513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205177.256517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205177.256518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205177.256646:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205177.256650:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205177.256652:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205177.576513:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205177.576520:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205177.576523:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205177.576528:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205177.576532:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205177.576536:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205177.576538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205177.576541:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205177.576543:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205178.024506:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205178.024513:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205178.024515:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205178.024519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205178.024523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205178.024527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205178.024529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205178.024531:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205178.024534:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205178.088602:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205178.088611:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205178.088613:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205178.088618:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205178.088622:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205178.088626:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205178.088629:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205178.088632:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205178.088635:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205178.088721:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205178.088725:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205178.088726:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205178.088729:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205178.088731:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205178.088733:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.088735:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205178.088736:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.088738:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1788205178.089579:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000400:00000100:1.0:1788205178.089627:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1788205178.089636:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205178.089640:0:2334:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:1.0:1788205178.089645:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205178.089649:0:2334:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1788205178.089654:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:1.0:1788205178.089658:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ec800. 00000400:00000200:1.0:1788205178.089663:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1788205178.089669:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205178.089675:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4330 00000400:00000010:1.0:1788205178.089677:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4330. 00000100:00000001:1.0:1788205178.089682:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205178.089690:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205178.089710:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205178.089714:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205178.089721:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205178.089724:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205178.089732:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:1.0:1788205178.089735:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000100:00000001:0.0:1788205178.089737:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000800:00000010:1.0:1788205178.089740:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c30a0ecb00. 00000100:00000001:0.0:1788205178.089741:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:1.0:1788205178.089746:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:0.0:1788205178.089753:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205178] req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205178.089754:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205178.089759:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4e58 00000400:00000010:1.0:1788205178.089763:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4e58. 00000100:00000001:1.0:1788205178.089768:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:0.0:1788205178.089772:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:1.0:1788205178.089774:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205178.089776:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205178.089780:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5c60. 00000400:00000200:0.0:1788205178.089787:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f43b8 00000100:00000001:1.0:1788205178.089788:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1788205178.089790:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f43b8. 00000100:00000040:1.0:1788205178.089795:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.089797:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205178.089804:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205178.089807:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205178.089812:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1788205178.089822:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000100:00000200:0.0:1788205178.089823:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1788205178.089827:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264300. 00000400:00000200:1.0:1788205178.089833:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1788205178.089837:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205178.089841:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205178.089842:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1788205178.089846:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1788205178.089847:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000100:00000001:0.0:1788205178.089849:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1788205178.089851:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00100000:0.0:1788205178.089857:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205178.089857:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205178.089864:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.089871:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205178.089879:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.089883:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205178.089887:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.089904:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205178.089907:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205178.089909:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205178.089911:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205178.089913:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.089915:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.089920:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000800:00000100:1.0:1788205178.089920:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000100:00000001:0.0:1788205178.089923:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000800:00000010:1.0:1788205178.089925:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000100:00080000:0.0:1788205178.089929:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:1.0:1788205178.089930:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:0.0:1788205178.089938:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000400:00000200:1.0:1788205178.089939:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1788205178.089945:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000100:00000001:0.0:1788205178.089946:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000010:1.0:1788205178.089948:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205178.089952:0:2334:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205178.089956:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205178.089964:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.089965:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205178.089972:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205178.089984:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205936256:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205178.089986:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205178.089991:0:2334:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205178.089993:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205178.089999:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205178] req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205178.090008:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205178.090009:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205178.090012:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5f20. 00000400:00000200:0.0:1788205178.090015:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4088 00000400:00000010:0.0:1788205178.090017:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4088. 00000100:00000001:0.0:1788205178.090019:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205178.090023:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205178.090036:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090044:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205178.090045:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090047:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090048:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205178.090051:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090058:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205178.090062:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090069:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205178.090071:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090072:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090073:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090075:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205178.090077:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205178.090080:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205178.090083:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205178.090086:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205178.090090:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205178.090098:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936384:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205178.090102:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205178.090105:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205178] req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205178.090118:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205178.090119:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205178.090121:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3066f5a50. 00000400:00000200:0.0:1788205178.090124:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3066f4aa0 00000400:00000010:0.0:1788205178.090125:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3066f4aa0. 00000100:00000001:0.0:1788205178.090128:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205178.090131:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205178.090138:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090144:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205178.090145:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090147:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090148:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205178.090152:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090159:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205178.090173:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090180:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205178.090182:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090183:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090185:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090186:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205178.090189:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205178.090191:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205178.090194:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205178.090197:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205178.090201:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205178.090208:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936512:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205178.090212:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205178.090215:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205178] req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205178.090223:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205178.090224:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205178.090226:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b1b8. 00000400:00000200:0.0:1788205178.090229:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657440 00000400:00000010:0.0:1788205178.090230:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657440. 00000100:00000001:0.0:1788205178.090232:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205178.090235:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205178.090244:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090250:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205178.090252:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090254:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090256:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205178.090260:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090268:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205178.090271:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090278:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205178.090281:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090283:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205178.090284:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090286:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205178.090288:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205178.090290:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205178.090293:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205178.090295:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205178.090298:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205178.090306:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936640:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205178.090310:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090312:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.090315:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205936256/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090322:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205178.090325:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205178.090329:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21c00. 02000000:00000001:0.0:1788205178.090333:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205178.090336:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205178.090339:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205178.090341:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205178.090344:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:0.0:1788205178.090348:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205178.090349:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205178.090351:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205178.090355:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205178.090356:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090358:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.090361:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205936384/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090393:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205178.090395:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205178.090397:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c21000. 02000000:00000001:0.0:1788205178.090399:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205178.090400:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205178.090402:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205178.090404:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205178.090405:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205178.090407:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205178.090408:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205178.090410:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205178.090412:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205178.090413:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090415:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.090418:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205936512/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090574:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205178.090576:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205178.090578:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310c22c00. 02000000:00000001:0.0:1788205178.090581:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205178.090583:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205178.090584:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205178.090586:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205178.090587:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205178.090590:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205178.090591:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205178.090592:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205178.090595:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205178.090596:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090597:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205178.090602:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205936640/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205178.090608:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205178.090609:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205178.090611:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:0.0:1788205178.090613:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205178.090615:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205178.090616:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205178.090618:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205178.090620:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205178.090622:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205178.090623:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205178.090625:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205178.090627:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205178.090628:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090630:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.090637:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205178.090638:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205178.090641:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205178.090642:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205178.280511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205178.280516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205178.280520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205178.280521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205178.280522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205178.280523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205178.280525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205178.280526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205178.280613:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205178.280615:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205178.280617:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205178.600543:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205178.600550:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205178.600552:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205178.600559:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205178.600563:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205178.600566:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.600569:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205178.600572:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205178.600574:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205179.049005:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205179.049013:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205179.049016:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205179.049021:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205179.049025:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205179.049028:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.049030:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205179.049033:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.049035:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205179.112427:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205179.112441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205179.112443:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205179.112447:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205179.112451:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205179.112456:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205179.112459:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205179.112462:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205179.112465:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205179.112511:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205179.112514:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205179.112516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205179.112519:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205179.112521:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205179.112523:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205179.112525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205179.112526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205179.112529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205179.304536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205179.304542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205179.304548:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205179.304549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205179.304552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205179.304554:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205179.304557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205179.304558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205179.307569:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205179.307574:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205179.307577:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205179.624487:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301306371, at 4301306880 00000100:00000010:1.0:1788205179.624503:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f41e0. 00000100:00000040:0.0:1788205179.628503:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301306371, at 4301306884 00000100:00000010:0.0:1788205179.628517:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3104f4000. 00000100:00000001:3.0:1788205179.630489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205179.630495:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205179.630497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205179.630501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205179.630504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205179.630508:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.630511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205179.630513:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.630516:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205180.072489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205180.072496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205180.072498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205180.072504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205180.072507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205180.072511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.072514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205180.072516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.072519:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.136551:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205180.136558:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205180.136560:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205180.136566:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205180.136569:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205180.136574:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.136577:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.136580:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.136583:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205180.136609:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.136616:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205180.136618:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205180.136621:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205180.136625:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205180.136631:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6634 00000400:00000200:0.0:1788205180.136638:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205180.136644:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205180.136647:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205180.136649:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136652:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.136654:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136657:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205180.136661:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205180.136662:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205180.136665:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205180.136668:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205180.136671:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205180.136673:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205180.136675:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136677:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.136679:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136681:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205180.136684:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205180.136687:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136689:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205180.136692:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205180.136695:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205180.136698:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.136700:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205180.136702:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205180.136708:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205180.136711:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205180.136715:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205180.136717:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.136719:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205180.136722:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205180.136725:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205180.136732:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8000. 00000100:00000001:0.0:1788205180.136738:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205180.136747:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205180.136757:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205180.136760:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205180.136765:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205180.136782:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205180.136784:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.136787:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205180.136788:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205180.136790:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205180.136792:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205180.136795:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6634 00000400:00000200:0.0:1788205180.136797:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205180.136800:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205180.136802:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205180.136805:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136807:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.136809:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136811:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205180.136813:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205180.136814:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205180.136815:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205180.136817:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205180.136819:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205180.136821:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205180.136824:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136826:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.136827:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136829:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205180.136831:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205180.136833:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136836:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205180.136838:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205180.136842:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205180.136844:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.136846:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205180.136850:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205180.136852:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.136854:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205180.136857:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205180.136860:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205180.136862:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205180.136863:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.136864:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205180.136866:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205180.136867:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205180.136870:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8800. 00000100:00000001:0.0:1788205180.136872:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205180.136877:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205180.136884:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205180.136886:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205180.136889:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205180.136904:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205180.136907:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.136910:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205180.136911:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205180.136913:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205180.136915:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205180.136918:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6634 00000400:00000200:0.0:1788205180.136921:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205180.136924:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205180.136926:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205180.136928:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136930:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.136932:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.136934:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205180.136936:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205180.136937:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:0.0:1788205180.136939:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205180.136940:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:2.0:1788205180.137138:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205180.137142:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205180.137145:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205180.137153:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.137162:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.137164:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.137168:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205180.137171:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205180.137174:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205180.137178:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205936768:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205180.137183:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205180.137187:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205180.137190:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.137193:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205180.137198:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205180.137201:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205180.137207:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205180.137211:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205180.137218:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205180.137230:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936768, portal 25 00000100:00000001:2.0:1788205180.137233:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205180.137235:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788205180.137242:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.137250:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205180.137254:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205180.137257:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205180.137261:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205936768, offset 0 00000400:00000200:2.0:1788205180.137269:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205180.137278:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.137286:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205180.137292:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205180.137297:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205180.137302:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205180.137306:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205180.137311:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59174:59174:256:4294967295] 192.168.201.151@tcp LPNI seq info [59154:59154:8:4294967295] 00000400:00000200:2.0:1788205180.137316:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.137325:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205180.137332:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.137336:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205180.137343:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205180.137347:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788205180.137352:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205180.137355:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788205180.137358:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.137361:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205180.137382:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205180.137387:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205180.137392:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205180.137394:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205180.137398:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205180.137403:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205180.137912:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205180.137916:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205180.137923:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.137927:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.137930:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.137932:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205180.137934:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205180.137937:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205180.137939:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205180.137942:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205180.137947:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205180.137950:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.137952:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205180.137955:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205180.137958:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.137960:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205180.137965:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205180.137968:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205180.137972:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205180.137974:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.137977:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205180.137979:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205180.137982:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205180.137987:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcbc00. 00000100:00000001:0.0:1788205180.137992:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205180.137999:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205180.138007:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205180.138010:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205180.138014:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205180.138025:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205180.138029:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.138033:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205180.138035:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205180.138037:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205180.138040:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205180.138043:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6634 00000400:00000200:0.0:1788205180.138049:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205180.138054:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205180.138056:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205180.138060:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.138062:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.138066:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.138068:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205180.138071:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205180.138072:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:0.0:1788205180.138074:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205180.138077:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:0.0:1788205180.138079:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205180.138082:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205180.138085:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.138088:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205180.138090:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205180.138092:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205180.138095:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205180.138098:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205180.138101:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205180.138103:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205180.138106:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205180.138108:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.138110:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205180.138112:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205180.138115:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.138117:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205180.138120:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205180.138123:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205180.138125:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205180.138127:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205180.138129:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205180.138131:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205180.138133:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205180.138137:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:0.0:1788205180.138141:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205180.138146:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205180.138154:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205180.138156:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205180.138160:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205180.138168:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205180.138173:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6645) 00000100:00000001:2.0:1788205180.138469:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.138477:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138487:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.138489:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205180.138500:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205180.138506:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205180.138509:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301311491 00000100:00000010:2.0:1788205180.138514:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1900. 00000100:00000001:2.0:1788205180.138522:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205180.138529:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301311491 00000100:00000010:2.0:1788205180.138532:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1a80. 00000100:00000001:2.0:1788205180.138536:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788205180.138541:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205180.138544:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.138547:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205180.138553:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205180.138564:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205180.138566:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205180.138569:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205180.138573:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205180.138575:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205180.138580:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205180.138584:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205180.138594:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138601:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205180.138603:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.138606:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.138608:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205180.138613:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138620:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205180.138626:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138633:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205180.138636:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.138638:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.138640:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.138642:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205180.138645:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205180.138648:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205180.138653:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205180.138656:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205180.138661:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205180.138669:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205936768:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205180.138676:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205180.138681:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138688:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.138690:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.138692:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205180.138695:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205180.138698:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205180.138702:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936896:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205180.138705:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205180.138709:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205180.138711:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.138714:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205180.138718:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205180.138720:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205180.138725:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205180.138728:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205180.138732:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205180.138737:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205936896, portal 10 00000100:00000001:2.0:1788205180.138739:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205180.138741:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788205180.138750:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138757:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205180.138761:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205180.138764:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205180.138768:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205936896, offset 0 00000400:00000200:2.0:1788205180.138774:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205180.138781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.138788:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205180.138794:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205180.138799:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205180.138804:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205180.138808:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205180.138813:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59175:59175:256:4294967295] 192.168.201.151@tcp LPNI seq info [59155:59155:8:4294967295] 00000400:00000200:2.0:1788205180.138818:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.138832:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205180.138839:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.138843:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205180.138850:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205180.138854:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788205180.138873:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205180.138876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788205180.138882:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.138884:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205180.138889:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205180.138895:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205180.138900:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205180.138902:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205180.138906:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205180.138911:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138920:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.138925:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.138932:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.138934:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205180.138940:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205180.138944:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788205180.138948:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205180.138951:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301307548 00000100:00000001:2.0:1788205180.138954:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.138956:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301307548 00000100:00000001:2.0:1788205180.138959:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.138961:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205180.138964:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.138966:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205180.138972:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205180.138981:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205180.138983:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205180.138985:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205180.138989:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205180.138991:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205180.138995:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205180.138999:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205180.139009:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139015:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205180.139017:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139019:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.139021:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205180.139026:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139034:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205180.139040:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139047:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205180.139050:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139052:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.139053:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139056:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205180.139060:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205180.139063:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205180.139068:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205180.139071:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205180.139076:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205180.139084:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205936896:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205180.139089:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139091:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.139096:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205936768/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139103:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205180.139106:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205180.139109:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205180.139113:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205180.139116:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205180.139118:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788205180.139120:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205180.139123:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8000. 02000000:00000001:2.0:1788205180.139128:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205180.139130:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205180.139132:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788205180.139136:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205180.139138:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139140:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.139144:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205936896/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139150:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205180.139152:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205180.139155:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205180.139158:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205180.139160:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205180.139167:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788205180.139170:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205180.139172:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8800. 02000000:00000001:2.0:1788205180.139175:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205180.139177:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205180.139178:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788205180.139180:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205180.139182:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139184:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139190:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205180.139192:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.139195:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205180.139197:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205180.139199:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205180.139203:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139208:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.139209:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.139211:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205180.139213:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205180.139214:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205180.139217:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205180.139220:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205180.139222:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205180.139223:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.139225:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205180.139227:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205180.139229:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205180.139233:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205180.139235:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205180.139238:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205180.139241:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937024, portal 4 00000100:00000001:2.0:1788205180.139243:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205180.139245:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788205180.139250:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139254:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205180.139257:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205180.139259:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205180.139261:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205937024, offset 0 00000400:00000200:2.0:1788205180.139265:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205180.139269:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.139273:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205180.139277:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205180.139280:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205180.139283:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205180.139285:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205180.139289:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59176:59176:256:4294967295] 192.168.201.151@tcp LPNI seq info [59156:59156:8:4294967295] 00000400:00000200:2.0:1788205180.139292:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.139301:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205180.139306:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.139308:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205180.139312:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205180.139315:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788205180.139319:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205180.139321:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788205180.139326:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.139327:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205180.139339:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205180.139343:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205180.139346:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205180.139348:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205180.139350:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205180.139354:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139359:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.139392:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139400:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.139402:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205180.139408:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205180.139421:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788205180.139426:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205180.139428:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301307435 00000100:00000001:2.0:1788205180.139431:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.139434:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301307435 00000100:00000001:2.0:1788205180.139436:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.139438:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205180.139441:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.139443:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205180.139449:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205180.139458:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205180.139460:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205180.139462:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205180.139466:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205180.139468:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205180.139471:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205180.139476:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205180.139485:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139491:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205180.139493:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139495:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.139496:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205180.139499:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139504:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205180.139508:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.139512:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205180.139514:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139516:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.139517:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.139518:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205180.139521:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205180.139523:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205180.139526:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205180.140720:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205180.140735:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205180.140744:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937024:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205180.140750:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205180.140753:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.140759:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.140761:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.140764:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205180.140767:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205180.140770:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205180.140773:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205180.140776:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205180.140780:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205180.140781:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205180.140783:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205180.140790:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205180.140792:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205180.140797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205180.140800:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205180.140806:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205180.140811:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937152, portal 4 00000100:00000001:2.0:1788205180.140813:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205180.140814:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:2.0:1788205180.140820:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.140826:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205180.140829:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205180.140831:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205180.140835:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205937152, offset 0 00000400:00000200:2.0:1788205180.140840:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205180.140891:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.140896:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205180.140899:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205180.140902:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205180.140904:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205180.140905:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205180.140908:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59177:59177:256:4294967295] 192.168.201.151@tcp LPNI seq info [59157:59157:8:4294967295] 00000400:00000200:2.0:1788205180.140910:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205180.140916:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205180.140919:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.140922:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205180.140926:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205180.140928:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1788205180.140931:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205180.140933:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1788205180.140936:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205180.140937:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205180.140941:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205180.140943:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205180.140946:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205180.140947:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205180.140950:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205180.140953:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.140958:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.140960:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.140964:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.140965:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205180.140968:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205180.140970:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788205180.140972:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205180.140974:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301307442 00000100:00000001:2.0:1788205180.140976:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.140977:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301307442 00000100:00000001:2.0:1788205180.140979:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301311491 00000100:00000001:2.0:1788205180.140981:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205180.140982:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.140984:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205180.140986:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205180.140991:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205180.140992:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205180.140993:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205180.140994:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205180.140995:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205180.140996:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205180.140998:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205180.141003:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.141007:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205180.141008:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141009:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.141010:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205180.141012:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.141017:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205180.141021:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.141027:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205180.141030:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141032:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205180.141033:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141036:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205180.141038:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205180.141041:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205180.141045:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205180.141047:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205180.141051:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205180.141059:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937152:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205180.141063:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141065:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.141068:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205937024/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.141074:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205180.141076:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205180.141078:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205180.141081:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205180.141083:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205180.141085:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788205180.141087:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205180.141089:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:2.0:1788205180.141092:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205180.141094:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205180.141095:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788205180.141099:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205180.141100:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141101:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205180.141104:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205937152/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205180.141110:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205180.141111:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205180.141113:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205180.141115:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205180.141116:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205180.141118:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788205180.141119:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205180.141121:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9000. 02000000:00000001:2.0:1788205180.141122:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205180.141123:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205180.141124:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788205180.141126:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205180.141127:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141129:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.141135:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205180.141137:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.141139:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205180.141140:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205180.142641:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205180.142646:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205180.142647:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205180.142651:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205180.142655:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205180.142658:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.142660:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205180.142662:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.142664:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205180.328552:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205180.328558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205180.328564:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205180.328565:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205180.328568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205180.328570:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205180.328573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205180.328574:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205180.328637:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205180.328640:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205180.328644:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205180.648453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205180.648461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205180.648464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205180.648469:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205180.648474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205180.648477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.648480:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205180.648483:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.648486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.096463:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205181.096472:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205181.096474:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.096480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205181.096484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205181.096489:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.096492:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205181.096495:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.096499:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.160525:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205181.160526:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205181.160531:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205181.160534:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205181.160535:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.160538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205181.160544:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205181.160545:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205181.160553:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205181.160555:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205181.160561:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205181.160562:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.160567:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205181.160572:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205181.160577:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205181.160581:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.160585:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.160586:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205181.352504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205181.352515:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205181.352518:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205181.352522:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352526:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205181.352528:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352531:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352534:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205181.352545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205181.352549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205181.352550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205181.352641:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205181.352648:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205181.352662:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205181.672537:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205181.672548:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205181.672551:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.672560:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205181.672565:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205181.672570:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.672573:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205181.672577:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.672580:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205182.121505:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205182.121511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205182.121513:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205182.121516:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205182.121519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205182.121522:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.121524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205182.121525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.121527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205182.184500:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205182.184509:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205182.184511:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205182.184517:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205182.184521:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205182.184526:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.184528:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205182.184530:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.184533:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205182.184897:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205182.184903:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205182.184905:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205182.184910:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205182.184913:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205182.184916:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205182.184919:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205182.184921:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205182.184924:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205182.376587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205182.376598:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205182.376607:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205182.376608:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205182.376612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205182.376613:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205182.376630:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205182.376631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205182.377063:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205182.377068:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205182.377071:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205182.696493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205182.696500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205182.696502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205182.696513:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205182.696517:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205182.696521:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.696524:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205182.696527:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.696530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205183.144606:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205183.144611:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205183.144613:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205183.144616:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205183.144618:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205183.144621:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.144623:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205183.144626:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.144628:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205183.208525:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205183.208535:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205183.208537:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205183.208543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205183.208547:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205183.208553:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205183.208556:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205183.208560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205183.208563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205183.208794:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205183.208802:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205183.208804:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205183.208810:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205183.208814:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205183.208817:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.208820:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205183.208823:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.208825:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205183.400575:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205183.400577:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205183.400581:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400586:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205183.400588:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400590:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400592:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400596:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205183.400597:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400600:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205183.400602:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205183.400603:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205183.400605:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400607:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205183.400608:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400609:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400611:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400615:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205183.400616:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205183.400617:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205183.400619:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400620:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205183.400621:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400632:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205183.400634:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205183.400716:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205183.400723:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205183.400728:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205183.720486:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205183.720494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205183.720496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205183.720501:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205183.720505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205183.720510:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.720512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205183.720514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.720517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205184.168478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205184.168484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205184.168485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205184.168489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205184.168491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205184.168495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.168497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205184.168499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.168501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205184.232483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205184.232492:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205184.232494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205184.232500:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205184.232505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205184.232509:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.232512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205184.232515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.232520:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205184.232591:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205184.232599:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205184.232602:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205184.232607:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205184.232610:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205184.232614:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205184.232617:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205184.232619:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205184.232622:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205184.424616:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205184.424624:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205184.424630:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205184.424631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205184.424635:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205184.424637:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205184.424639:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205184.424641:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205184.424696:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205184.424699:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205184.424702:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205184.744492:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301311491, at 4301312000 00000100:00000040:1.0:1788205184.744492:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301311491, at 4301312000 00000100:00000001:3.0:1788205184.744498:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205184.744505:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205184.744509:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205184.744510:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1a80. 00000100:00000010:0.0:1788205184.744513:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1900. 00000100:00100000:3.0:1788205184.744521:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205184.744525:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205184.744528:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.744531:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205184.744534:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.744538:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205185.192485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205185.192493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205185.192496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205185.192502:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205185.192506:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205185.192510:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.192512:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205185.192515:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.192518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256505:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205185.256505:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205185.256512:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205185.256513:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205185.256514:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256516:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205185.256520:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788205185.256521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205185.256523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205185.256527:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205185.256528:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.256531:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.256531:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205185.256535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.256538:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205185.256538:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256542:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.256545:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205185.256625:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205185.256632:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205185.256635:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205185.256638:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205185.256642:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205185.256649:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6640 00000400:00000200:2.0:1788205185.256657:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205185.256663:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205185.256666:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205185.256669:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256672:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205185.256675:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256677:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205185.256681:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205185.256683:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205185.256686:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205185.256689:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205185.256691:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205185.256693:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205185.256696:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256697:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205185.256699:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256701:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205185.256704:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205185.256707:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256710:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205185.256713:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205185.256716:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205185.256719:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.256721:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205185.256723:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205185.256732:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205185.256735:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205185.256739:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205185.256741:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.256744:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205185.256746:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205185.256750:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205185.256757:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788205185.256763:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205185.256772:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205185.256782:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205185.256785:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205185.256790:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205185.256813:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205185.256816:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205185.256823:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205185.256825:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205185.256826:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205185.256829:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205185.256832:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6640 00000400:00000200:2.0:1788205185.256836:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205185.256839:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205185.256841:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788205185.256841:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205185.256844:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205185.256844:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256845:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205185.256846:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205185.256847:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256849:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:0.0:1788205185.256851:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205185.256851:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205185.256853:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205185.256854:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205185.256856:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205185.256858:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205185.256859:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788205185.256860:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205185.256861:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205185.256863:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205185.256864:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205185.256865:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205185.256867:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788205185.256867:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205185.256869:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.256869:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205185.256871:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00100000:0.0:1788205185.256873:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205937280:192.168.201.151@tcp:250:kworker.0 00000100:00080000:2.0:1788205185.256874:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256876:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256877:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205185.256877:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205185.256880:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205185.256882:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205185.256883:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205185.256885:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.256886:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205185.256886:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205185.256890:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 02000000:00000010:0.0:1788205185.256891:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205185.256893:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205185.256893:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.256895:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.256898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:2.0:1788205185.256898:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205185.256900:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:0.0:1788205185.256902:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 02000000:00000001:2.0:1788205185.256903:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205185.256904:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.256905:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205185.256906:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1788205185.256907:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 02000000:00000001:2.0:1788205185.256908:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205185.256911:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000200:0.0:1788205185.256914:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937280, portal 25 00000100:00000001:2.0:1788205185.256914:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256916:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205185.256917:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00100000:2.0:1788205185.256919:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1788205185.256922:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205185.256926:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205185.256928:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.256929:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205185.256932:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788205185.256932:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788205185.256935:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205185.256937:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205937280, offset 0 00000100:00000001:2.0:1788205185.256940:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205185.256942:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1788205185.256943:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205185.256945:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205185.256947:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205185.256948:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205185.256950:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205185.256952:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6640 00000400:00000200:2.0:1788205185.256955:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205185.256958:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205185.256960:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205185.256962:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205185.256963:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205185.256964:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205185.256966:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256968:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205185.256969:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000001:2.0:1788205185.256970:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205185.256971:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205185.256972:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:0.0:1788205185.256974:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1788205185.256974:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205185.256976:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205185.256978:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:2.0:1788205185.256978:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205185.256980:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205185.256982:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788205185.256982:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205185.256984:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000040:2.0:1788205185.256984:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.256986:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205185.256988:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59178:59178:256:4294967295] 192.168.201.151@tcp LPNI seq info [59158:59158:8:4294967295] 00000020:00000040:2.0:1788205185.256988:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205185.256990:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:0.0:1788205185.256993:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205185.256993:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205185.256994:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205185.256997:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205185.256999:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.257001:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205185.257001:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205185.257003:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205185.257005:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205185.257007:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205185.257007:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205185.257010:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000800:00000010:0.0:1788205185.257012:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000020:00000040:2.0:1788205185.257012:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205185.257014:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205185.257015:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.257016:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1788205185.257018:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788205185.257018:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205185.257019:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205185.257022:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 02000000:00000010:2.0:1788205185.257022:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788205185.257025:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788205185.257027:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00100000:2.0:1788205185.257029:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1788205185.257030:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205185.257035:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205185.257036:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1788205185.257037:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000100:00000001:2.0:1788205185.257038:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205185.257041:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205185.257045:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205185.257048:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.257049:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1788205185.257050:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205185.257053:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:0.0:1788205185.257054:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000100:00000001:2.0:1788205185.257055:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1788205185.257056:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00080000:2.0:1788205185.257056:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205185.257059:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205185.257061:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788205185.257061:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6640 00000100:00000200:0.0:1788205185.257064:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205185.257064:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205185.257066:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205185.257069:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205185.257071:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205185.257073:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205185.257073:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205185.257075:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205185.257076:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205185.257077:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205185.257079:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205185.257081:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205185.257082:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788205185.257084:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205185.257084:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:0.0:1788205185.257086:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1788205185.257086:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205185.257088:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205185.257090:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205185.257091:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.257092:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205185.257094:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788205185.257094:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205185.257096:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205185.257097:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301316609 00000020:00000040:2.0:1788205185.257098:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205185.257100:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000010:0.0:1788205185.257101:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9180. 00000100:00000001:2.0:1788205185.257102:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205185.257104:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205185.257106:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205185.257107:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788205185.257108:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.257109:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205185.257111:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000100:00000001:0.0:1788205185.257112:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301316609 00000008:00000001:2.0:1788205185.257113:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205185.257115:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9f00. 00000100:00000001:2.0:1788205185.257115:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257117:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000010:2.0:1788205185.257119:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205185.257120:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1788205185.257120:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:0.0:1788205185.257123:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205185.257123:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205185.257124:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257125:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205185.257125:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205185.257127:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205185.257128:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205185.257129:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205185.257131:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205185.257134:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788205185.257137:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788205185.257138:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788205185.257139:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205185.257141:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205185.257143:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205185.257145:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:2.0:1788205185.257145:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205185.257147:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788205185.257147:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205185.257151:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205185.257151:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205185.257159:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205185.257160:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205185.257162:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6650) 00000100:00000001:0.0:1788205185.257166:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205185.257168:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257170:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257172:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205185.257176:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257182:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205185.257187:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257194:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205185.257196:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257198:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257199:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257201:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205185.257203:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205185.257205:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205185.257209:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205185.257212:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205185.257216:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205185.257224:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205937280:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205185.257228:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257230:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.257233:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205937280/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257244:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205185.257246:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205185.257249:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205185.257252:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205185.257254:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205185.257255:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205185.257257:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205185.257260:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:0.0:1788205185.257264:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205185.257265:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205185.257267:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205185.257270:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205185.257271:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257273:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257280:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205185.257282:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257284:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205185.257285:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205185.257287:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205185.257290:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257297:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205185.257298:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257301:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205185.257302:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205185.257305:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205185.257308:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937408:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205185.257310:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205185.257312:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205185.257314:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257316:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205185.257319:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205185.257320:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.257381:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205185.257386:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205185.257390:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205185.257393:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937408, portal 10 00000100:00000001:0.0:1788205185.257395:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205185.257396:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205185.257403:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257410:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205185.257412:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205185.257414:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205185.257416:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205937408, offset 0 00000400:00000200:0.0:1788205185.257420:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257424:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.257428:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205185.257430:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205185.257433:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205185.257435:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205185.257438:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257440:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59179:59179:256:4294967295] 192.168.201.151@tcp LPNI seq info [59159:59159:8:4294967295] 00000400:00000200:0.0:1788205185.257444:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.257450:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205185.257454:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.257458:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205185.257461:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205185.257464:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205185.257468:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205185.257470:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205185.257473:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.257475:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205185.257479:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205185.257482:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205185.257484:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205185.257486:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205185.257489:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205185.257492:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257499:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.257502:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257507:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257509:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205185.257512:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205185.257515:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205185.257519:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205185.257520:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301312667 00000100:00000001:0.0:1788205185.257523:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.257525:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301312667 00000100:00000001:0.0:1788205185.257527:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.257529:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205185.257530:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257532:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205185.257535:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205185.257542:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205185.257543:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205185.257544:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205185.257546:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205185.257548:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205185.257550:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205185.257552:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205185.257559:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257570:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205185.257571:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257573:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257574:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205185.257577:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257583:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205185.257586:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257592:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205185.257594:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257596:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257597:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257599:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205185.257601:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205185.257604:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205185.257606:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205185.257609:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205185.257612:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205185.257619:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937408:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205185.257622:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205185.257625:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257631:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205185.257632:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257634:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205185.257636:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205185.257638:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205185.257640:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937536:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205185.257643:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205185.257645:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205185.257646:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257647:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205185.257650:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205185.257652:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.257655:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205185.257657:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205185.257660:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205185.257662:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937536, portal 4 00000100:00000001:0.0:1788205185.257664:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205185.257665:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205185.257670:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257675:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205185.257677:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205185.257679:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205185.257681:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205937536, offset 0 00000400:00000200:0.0:1788205185.257684:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257687:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.257690:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205185.257700:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205185.257703:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205185.257705:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205185.257707:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257709:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59180:59180:256:4294967295] 192.168.201.151@tcp LPNI seq info [59160:59160:8:4294967295] 00000400:00000200:0.0:1788205185.257713:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.257718:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205185.257723:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.257726:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205185.257730:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205185.257732:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205185.257736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205185.257738:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205185.257741:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.257743:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205185.257746:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205185.257758:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205185.257761:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205185.257763:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205185.257765:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205185.257768:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257775:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.257777:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257783:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257784:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205185.257787:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205185.257789:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205185.257792:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205185.257794:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301312553 00000100:00000001:0.0:1788205185.257796:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.257798:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301312553 00000100:00000001:0.0:1788205185.257800:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.257802:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205185.257804:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.257806:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205185.257810:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205185.257817:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205185.257819:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205185.257820:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205185.257823:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205185.257824:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205185.257826:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205185.257829:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205185.257843:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257849:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205185.257850:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257852:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257853:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205185.257857:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257863:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205185.257867:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257872:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205185.257875:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257877:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.257878:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.257880:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205185.257882:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205185.257884:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205185.257888:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205185.257890:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205185.257895:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205185.257902:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937536:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205185.257905:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205185.257908:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257915:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205185.257917:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257919:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205185.257920:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205185.257922:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205185.257925:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937664:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205185.257928:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205185.257930:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205185.257931:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205185.257933:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205185.257936:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205185.257938:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205185.257941:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205185.257943:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205185.257946:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205185.257949:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937664, portal 4 00000100:00000001:0.0:1788205185.257951:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205185.257952:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205185.257957:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.257963:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205185.257966:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205185.257968:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205185.257970:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205937664, offset 0 00000400:00000200:0.0:1788205185.257974:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257978:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.257981:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205185.257984:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205185.257986:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205185.257989:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205185.257991:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205185.257994:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59181:59181:256:4294967295] 192.168.201.151@tcp LPNI seq info [59161:59161:8:4294967295] 00000400:00000200:0.0:1788205185.257998:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205185.258004:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205185.258008:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.258011:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205185.258015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205185.258017:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788205185.258021:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205185.258024:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788205185.258028:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205185.258029:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205185.258032:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205185.258035:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205185.258038:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205185.258040:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205185.258042:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205185.258046:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258053:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.258056:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258062:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.258064:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205185.258067:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205185.258069:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205185.258072:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205185.258075:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301312559 00000100:00000001:0.0:1788205185.258077:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.258079:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301312559 00000100:00000001:0.0:1788205185.258081:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301316609 00000100:00000001:0.0:1788205185.258082:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205185.258084:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.258086:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205185.258089:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205185.258096:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205185.258097:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205185.258098:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205185.258100:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205185.258102:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205185.258103:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205185.258107:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205185.258115:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258121:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205185.258122:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258123:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.258125:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205185.258128:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258134:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205185.258138:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258144:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205185.258147:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258148:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205185.258149:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258151:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205185.258153:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205185.258155:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205185.258158:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205185.258160:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205185.258163:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205185.258170:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937664:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205185.258173:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258176:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.258179:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205937408/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258185:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205185.258186:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205185.258188:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:0.0:1788205185.258190:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205185.258192:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205185.258193:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205185.258195:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205185.258197:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:0.0:1788205185.258199:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205185.258201:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205185.258202:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205185.258204:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205185.258206:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258207:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.258211:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205937536/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258222:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205185.258223:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205185.258224:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205185.258227:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205185.258228:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205185.258229:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205185.258231:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205185.258232:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:0.0:1788205185.258234:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205185.258235:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205185.258236:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205185.258238:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205185.258239:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258240:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205185.258243:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205937664/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205185.258248:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205185.258249:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205185.258251:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205185.258252:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205185.258253:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205185.258254:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205185.258256:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205185.258257:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205185.258258:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205185.258259:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205185.258260:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205185.258262:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205185.258263:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258265:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205185.258269:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205185.258270:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205185.258272:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205185.258274:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205185.448797:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205185.448804:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205185.448810:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205185.448812:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205185.448815:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205185.448817:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205185.448821:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205185.448822:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205185.448908:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205185.448912:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205185.448916:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205185.768907:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205185.768914:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205185.768917:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205185.768922:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205185.768926:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205185.768930:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.768933:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205185.768935:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.768939:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205186.217139:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205186.217148:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205186.217150:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205186.217155:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205186.217159:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205186.217163:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.217166:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205186.217169:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.217172:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205186.280488:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205186.280494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205186.280495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205186.280499:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205186.280501:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205186.280504:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205186.280507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205186.280509:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205186.280511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205186.280797:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205186.280801:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205186.280802:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205186.280805:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205186.280807:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205186.280809:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.280811:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205186.280812:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.280814:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205186.472514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205186.472519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205186.472525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205186.472526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205186.472529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205186.472530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205186.472534:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205186.472535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205186.472600:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205186.472603:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205186.472605:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205186.792481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205186.792488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205186.792491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205186.792496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205186.792500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205186.792503:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.792506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205186.792509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.792512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205187.240492:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205187.240500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205187.240503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205187.240508:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205187.240512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205187.240516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.240519:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205187.240522:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.240524:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205187.304499:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205187.304507:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205187.304509:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205187.304515:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205187.304519:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205187.304523:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.304526:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205187.304528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.304531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205187.305386:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205187.305390:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205187.305392:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205187.305395:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205187.305397:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205187.305399:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205187.305402:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205187.305403:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205187.305405:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205187.496479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205187.496485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205187.496490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205187.496491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205187.496493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205187.496494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205187.496497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205187.496498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205187.496661:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205187.496664:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205187.496666:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205187.816460:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205187.816467:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205187.816470:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205187.816475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205187.816479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205187.816483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.816485:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205187.816488:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.816491:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205188.264489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205188.264496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205188.264498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205188.264503:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205188.264505:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205188.264508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.264510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205188.264512:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.264513:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205188.328481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205188.328487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205188.328489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205188.328492:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205188.328493:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205188.328496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205188.328499:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788205188.328502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205188.328505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205188.328506:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205188.328510:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.328510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205188.328515:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205188.328518:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205188.328520:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205188.328522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205188.328525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.328528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205188.520657:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205188.520664:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205188.520669:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205188.520670:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205188.520674:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205188.520675:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205188.520678:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205188.520679:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205188.521396:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205188.521400:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205188.521403:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205188.842398:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205188.842405:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205188.842407:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205188.842413:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205188.842416:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205188.842420:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.842422:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205188.842425:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.842427:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205189.288468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205189.288474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205189.288476:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205189.288479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205189.288481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205189.288484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205189.288486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205189.288487:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205189.288489:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205189.352457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205189.352464:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205189.352467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205189.352472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788205189.352473:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205189.352475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205189.352479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205189.352479:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205189.352481:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205189.352485:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205189.352485:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205189.352487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205189.352490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205189.352494:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205189.352498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205189.352498:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205189.352501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205189.352504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205189.544464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205189.544469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205189.544472:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205189.544474:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205189.544476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205189.544477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205189.544479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205189.544480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205189.544537:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205189.544541:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205189.544545:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205189.864453:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301316609, at 4301317120 00000100:00000001:3.0:1788205189.864453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1788205189.864455:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301316609, at 4301317120 00000100:00000001:3.0:1788205189.864458:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205189.864463:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205189.864467:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9180. 00000100:00100000:3.0:1788205189.864467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205189.864468:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9f00. 00000100:00000001:3.0:1788205189.864473:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205189.864477:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205189.864479:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205189.864481:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205189.864484:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205190.312464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205190.312470:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205190.312472:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205190.312475:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205190.312478:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205190.312480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.312482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205190.312483:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.312485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.376434:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205190.376434:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788205190.376436:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205190.376441:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205190.376441:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205190.376445:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205190.376445:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205190.376447:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1788205190.376452:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205190.376452:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205190.376452:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205190.376458:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205190.376459:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205190.376459:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205190.376463:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.376463:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205190.376465:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205190.376468:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205190.376470:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.376473:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205190.376474:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6645 00000100:00000001:3.0:1788205190.376475:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205190.376478:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.376480:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205190.376486:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205190.376495:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205190.376498:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205190.376501:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376504:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205190.376506:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376508:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205190.376512:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205190.376514:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205190.376517:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205190.376520:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205190.376522:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205190.376524:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205190.376527:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376529:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205190.376531:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376533:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205190.376536:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205190.376539:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376542:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205190.376545:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205190.376547:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205190.376550:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.376552:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205190.376554:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205190.376562:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205190.376566:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205190.376569:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205190.376571:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.376574:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205190.376576:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205190.376580:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205190.376587:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205190.376593:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205190.376602:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205190.376611:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205190.376615:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205190.376619:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.376639:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205190.376642:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205190.376646:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205190.376647:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205190.376650:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205190.376653:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205190.376655:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6645 00000100:00000001:0.0:1788205190.376658:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1788205190.376661:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205190.376662:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205190.376665:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1788205190.376668:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:0.0:1788205190.376672:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.376680:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205190.376680:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205190.376682:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.376686:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1788205190.376687:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205190.376689:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205190.376692:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.376692:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205190.376699:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00100000:0.0:1788205190.376700:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205937792:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205190.376706:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205190.376707:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205190.376710:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1788205190.376711:0:30130:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1788205190.376713:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205190.376715:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 02000000:00000001:0.0:1788205190.376716:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788205190.376719:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000010:0.0:1788205190.376720:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205190.376723:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205190.376723:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205190.376728:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000400:00000200:0.0:1788205190.376730:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205190.376735:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000020:00000001:2.0:1788205190.376735:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000400:00000010:0.0:1788205190.376743:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000040:2.0:1788205190.376743:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376748:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1788205190.376750:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937792, portal 25 00000100:00000040:2.0:1788205190.376752:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205190.376754:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205190.376758:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000001:2.0:1788205190.376758:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205190.376764:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000040:0.0:1788205190.376766:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205190.376772:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376775:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205190.376777:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205190.376784:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205190.376785:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788205190.376789:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205190.376790:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205190.376794:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000001:2.0:1788205190.376795:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:0.0:1788205190.376796:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205937792, offset 0 00000008:00100000:2.0:1788205190.376801:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1788205190.376802:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000008:00000001:2.0:1788205190.376807:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.376809:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.376810:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205190.376817:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000010:2.0:1788205190.376817:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205190.376823:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:0.0:1788205190.376825:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205190.376830:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 02000000:00000001:2.0:1788205190.376830:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205190.376834:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.376835:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788205190.376839:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1788205190.376840:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:2.0:1788205190.376843:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1788205190.376845:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59182:59182:256:4294967295] 192.168.201.151@tcp LPNI seq info [59162:59162:8:4294967295] 02000000:00000001:2.0:1788205190.376847:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.376850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788205190.376856:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000400:00000200:0.0:1788205190.376858:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205190.376862:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205190.376865:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.376869:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000100:00100000:2.0:1788205190.376872:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1788205190.376878:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205190.376883:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788205190.376887:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1788205190.376890:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205190.376892:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205190.376897:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788205190.376905:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205190.376913:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205190.376915:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205190.376916:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000100:00080000:2.0:1788205190.376920:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1788205190.376924:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1788205190.376925:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205190.376928:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1788205190.376930:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1788205190.376932:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1788205190.376935:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000100:00000001:2.0:1788205190.376936:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:0.0:1788205190.376937:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205190.376941:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788205190.376942:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6645 00000100:00000200:0.0:1788205190.376947:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205190.376952:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205190.376959:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205190.376965:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205190.376970:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205190.376971:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205190.376977:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.376977:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205190.376983:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205190.376986:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.376988:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205190.376989:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1788205190.376993:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205190.376995:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205190.376996:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205190.377000:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:0.0:1788205190.377001:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205190.377005:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301321729 00000020:00000040:2.0:1788205190.377005:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000010:0.0:1788205190.377011:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9180. 00000020:00000001:2.0:1788205190.377011:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205190.377017:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:0.0:1788205190.377018:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1788205190.377024:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205190.377025:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301321729 00000100:00000010:0.0:1788205190.377029:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9d20. 00000100:00000001:2.0:1788205190.377030:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205190.377032:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1788205190.377035:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205190.377037:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205190.377039:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205190.377040:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377044:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:2.0:1788205190.377046:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00100000:0.0:1788205190.377049:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205190/real 1788205190] req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205190.377053:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000001:0.0:1788205190.377059:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788205190.377059:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788205190.377061:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788205190.377064:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1788205190.377065:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205190.377069:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205190.377070:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 02000000:00000001:2.0:1788205190.377071:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205190.377072:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205190.377077:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205190.377077:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.377082:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205190.377086:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000200:0.0:1788205190.377087:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788205190.377092:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.377098:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377099:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205190.377101:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1788205190.377103:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205190.377104:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377106:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788205190.377109:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00100000:0.0:1788205190.377110:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205190.377115:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788205190.377117:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205190.377118:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205190.377122:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.377123:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205190.377126:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205190.377129:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377131:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205190.377134:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788205190.377135:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:0.0:1788205190.377136:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377138:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377141:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205190.377142:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377143:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205190.377148:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205190.377152:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00100000:2.0:1788205190.377153:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205190.377154:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205190.377160:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.377176:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1788205190.377177:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205937792:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205190.377182:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377183:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377187:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377195:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205937792/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205190.377195:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377202:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205190.377206:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377208:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205190.377212:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788205190.377212:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205190.377214:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205190.377217:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000100:00080000:2.0:1788205190.377217:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1788205190.377221:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205190.377223:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 00000100:00080000:2.0:1788205190.377224:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:0.0:1788205190.377227:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788205190.377228:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1788205190.377230:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1788205190.377232:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:0.0:1788205190.377233:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205190.377252:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205190.377252:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377254:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377259:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205190.377260:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377261:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205190.377262:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205190.377263:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205190.377268:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.377268:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205190.377275:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6645 00000100:00000001:0.0:1788205190.377277:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205190.377280:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377282:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1788205190.377284:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:0.0:1788205190.377285:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205190.377290:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205190.377293:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937920:192.168.201.151@tcp:38:kworker.0 00000400:00000200:2.0:1788205190.377294:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205190.377296:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205190.377299:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788205190.377300:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1788205190.377301:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377304:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377307:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb400. 00000100:00000040:2.0:1788205190.377308:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205190.377309:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.377314:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205190.377315:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205190.377318:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000100:00000040:2.0:1788205190.377319:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:0.0:1788205190.377321:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000001:2.0:1788205190.377323:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:0.0:1788205190.377325:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205937920, portal 10 00000020:00000001:2.0:1788205190.377327:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205190.377328:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1788205190.377330:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:0.0:1788205190.377331:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000020:00000001:2.0:1788205190.377336:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:0.0:1788205190.377339:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205190.377340:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205190.377346:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205190.377351:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:0.0:1788205190.377352:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205190.377357:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.377360:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205190.377362:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205190.377383:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205190.377386:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205190.377389:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:0.0:1788205190.377394:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205190.377395:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205190.377398:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 02000000:00000001:2.0:1788205190.377400:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1788205190.377403:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205937920, offset 0 02000000:00000001:2.0:1788205190.377405:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205190.377411:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205190.377411:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.377415:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1788205190.377418:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:2.0:1788205190.377420:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:0.0:1788205190.377425:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00000001:2.0:1788205190.377426:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.377431:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788205190.377431:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.377436:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000010:2.0:1788205190.377438:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000400:00000200:0.0:1788205190.377439:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205190.377442:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:2.0:1788205190.377445:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:0.0:1788205190.377446:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59183:59183:256:4294967295] 192.168.201.151@tcp LPNI seq info [59163:59163:8:4294967295] 00000400:00000200:0.0:1788205190.377452:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205190.377453:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205190.377457:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.377459:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205190.377461:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1788205190.377465:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205190.377466:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:0.0:1788205190.377469:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 02000000:00000001:2.0:1788205190.377470:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205190.377475:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205190.377478:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 02000000:00000010:2.0:1788205190.377478:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000800:00000200:0.1:1788205190.377484:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205190.377485:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788205190.377486:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205190.377494:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.377495:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000100:00100000:2.0:1788205190.377495:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205190.377499:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205190.377506:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205190.377510:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000100:00000001:2.0:1788205190.377511:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1788205190.377513:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:2.0:1788205190.377516:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377517:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205190.377522:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205190.377525:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377534:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377539:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205190.377543:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205190.377550:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6655) 00000100:00000001:0.0:1788205190.377552:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377554:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205190.377557:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205190.377559:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205190.377562:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205190.377563:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301317787 00000100:00000001:0.0:1788205190.377565:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377567:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301317787 00000100:00000001:0.0:1788205190.377568:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377569:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205190.377571:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377572:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205190.377574:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205190/real 1788205190] req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205190.377578:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205190.377579:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205190.377580:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205190.377582:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205190.377583:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205190.377584:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205190.377586:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205190.377590:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377594:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205190.377595:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377595:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377596:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205190.377598:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377602:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205190.377605:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377609:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205190.377610:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377611:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377611:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377612:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205190.377614:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205190.377615:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205190.377617:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205190.377618:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205190.377620:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205190.377624:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205937920:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205190.377626:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205190.377628:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377632:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205190.377633:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377634:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205190.377635:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205190.377636:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205190.377637:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938048:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205190.377639:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205190.377640:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205190.377641:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377642:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377644:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205190.377645:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.377647:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205190.377649:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205190.377651:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205190.377652:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938048, portal 4 00000100:00000001:0.0:1788205190.377653:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205190.377654:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788205190.377658:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377661:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205190.377662:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205190.377664:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205190.377665:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205938048, offset 0 00000400:00000200:0.0:1788205190.377667:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205190.377669:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205190.377671:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205190.377673:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205190.377674:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205190.377676:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205190.377677:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205190.377679:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59184:59184:256:4294967295] 192.168.201.151@tcp LPNI seq info [59164:59164:8:4294967295] 00000400:00000200:0.0:1788205190.377681:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205190.377685:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205190.377687:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.377688:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205190.377690:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205190.377692:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788205190.377694:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205190.377696:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205190.377698:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.377699:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205190.377701:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205190.377703:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205190.377705:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205190.377706:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205190.377707:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205190.377709:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377713:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377715:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377718:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377719:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205190.377721:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205190.377722:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205190.377724:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205190.377725:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301317673 00000100:00000001:0.0:1788205190.377726:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377727:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301317673 00000100:00000001:0.0:1788205190.377729:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377730:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205190.377731:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377732:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205190.377734:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205190/real 1788205190] req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205190.377737:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205190.377738:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205190.377739:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205190.377740:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205190.377741:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205190.377742:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205190.377744:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205190.377748:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377752:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205190.377752:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377753:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377754:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205190.377756:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377759:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205190.377761:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377765:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205190.377766:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377767:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377767:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377768:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205190.377769:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205190.377771:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205190.377772:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205190.377774:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205190.377776:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205190.377780:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938048:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205190.377782:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377783:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377785:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205937920/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377788:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205190.377789:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377791:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb400. 02000000:00000001:0.0:1788205190.377792:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205190.377793:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205190.377793:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205190.377795:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205190.377796:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:0.0:1788205190.377797:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205190.377798:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205190.377799:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205190.377801:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205190.377802:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377803:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377804:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205938048/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377808:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205190.377808:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377809:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205190.377810:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205190.377810:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205190.377811:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205190.377812:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205190.377813:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:0.0:1788205190.377814:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205190.377814:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205190.377815:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205190.377816:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205190.377817:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377818:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377820:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205190.377821:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377822:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205190.377822:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205190.377823:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205190.377825:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377829:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205190.377830:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377831:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205190.377832:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205190.377833:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205190.377834:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938176:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205190.377836:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205190.377837:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205190.377838:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205190.377839:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377841:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205190.377842:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205190.377843:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205190.377845:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205190.377846:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205190.377848:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938176, portal 4 00000100:00000001:0.0:1788205190.377849:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205190.377849:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788205190.377852:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377856:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205190.377857:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205190.377858:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205190.377859:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205938176, offset 0 00000400:00000200:0.0:1788205190.377861:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205190.377863:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205190.377865:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205190.377866:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205190.377868:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205190.377869:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205190.377871:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205190.377873:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59185:59185:256:4294967295] 192.168.201.151@tcp LPNI seq info [59165:59165:8:4294967295] 00000400:00000200:0.0:1788205190.377875:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205190.377878:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205190.377881:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.377882:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205190.377884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205190.377886:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788205190.377888:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205190.377889:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788205190.377891:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205190.377893:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205190.377894:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205190.377896:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205190.377898:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205190.377899:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205190.377900:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205190.377902:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377906:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377908:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377912:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377912:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205190.377914:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205190.377915:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205190.377917:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205190.377918:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301317679 00000100:00000001:0.0:1788205190.377919:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377921:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301317679 00000100:00000001:0.0:1788205190.377922:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301321729 00000100:00000001:0.0:1788205190.377924:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205190.377925:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377926:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205190.377927:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205190/real 1788205190] req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205190.377931:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205190.377932:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205190.377933:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205190.377934:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205190.377935:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205190.377936:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205190.377937:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205190.377941:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377945:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205190.377946:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377947:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377948:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205190.377949:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377953:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205190.377955:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377958:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205190.377959:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377960:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205190.377961:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377962:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205190.377963:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205190.377964:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205190.377966:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205190.377968:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205190.377970:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205190.377974:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938176:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205190.377975:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377976:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205190.377978:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205938176/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205190.377981:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205190.377981:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205190.377982:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205190.377983:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205190.377984:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205190.377985:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205190.377986:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205190.377986:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205190.377987:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205190.377988:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205190.377989:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205190.377990:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205190.377991:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377992:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205190.377993:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205190.377994:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205190.377995:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205190.377996:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205190.568501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205190.568506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205190.568510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205190.568510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205190.568512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205190.568513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205190.568515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205190.568516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205190.568570:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205190.568574:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205190.568578:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205190.888466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205190.888472:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205190.888473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205190.888477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205190.888479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205190.888482:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.888484:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205190.888486:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.888488:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.336453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205191.336459:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205191.336460:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205191.336464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205191.336467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205191.336469:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.336471:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205191.336472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.336474:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.400429:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205191.400437:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205191.400439:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205191.400444:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205191.400448:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205191.400452:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.400455:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205191.400458:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.400461:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.400489:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205191.400491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205191.400493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205191.400495:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205191.400498:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205191.400500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.400502:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205191.400503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.400505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592444:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205191.592449:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592452:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205191.592455:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205191.592456:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205191.592459:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592461:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205191.592462:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592463:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592465:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205191.592468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205191.592470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205191.592471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205191.592510:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205191.592513:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205191.592515:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205191.912496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205191.912508:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205191.912510:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205191.912521:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205191.912525:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205191.912528:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.912531:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205191.912533:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205191.912536:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.360485:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205192.360492:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205192.360494:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.360499:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205192.360503:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205192.360506:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.360509:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205192.360511:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.360514:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.425167:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205192.425177:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205192.425180:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.425189:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205192.425193:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205192.425196:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.425214:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205192.425218:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.425222:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.425224:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205192.425229:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205192.425230:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.425234:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205192.425237:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205192.425240:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.425242:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205192.425243:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.425245:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205192.616488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205192.616493:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205192.616496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205192.616497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205192.616499:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205192.616500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205192.616502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205192.616502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205192.616555:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205192.616559:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205192.616562:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205192.936464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205192.936469:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205192.936471:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.936474:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205192.936477:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205192.936480:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.936482:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205192.936484:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.936486:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205193.384480:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205193.384488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205193.384490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205193.384495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205193.384499:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205193.384502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.384505:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205193.384507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.384509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205193.448458:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205193.448458:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205193.448465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205193.448465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205193.448467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205193.448467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205193.448472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205193.448472:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205193.448475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205193.448475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205193.448479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.448479:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.448481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205193.448483:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.448486:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205193.448487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205193.448488:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205193.448490:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205193.640488:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205193.640491:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205193.640493:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640497:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205193.640500:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640502:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640512:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205193.640517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205193.640521:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205193.640522:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205193.640524:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640525:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205193.640527:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640528:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640530:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205193.640534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205193.640536:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205193.640537:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640539:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205193.640540:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640542:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205193.640543:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205193.640599:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205193.640604:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205193.640607:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205193.960492:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205193.960500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205193.960502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205193.960507:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205193.960510:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205193.960514:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.960518:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205193.960521:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205193.960524:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.408520:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205194.408527:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205194.408529:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205194.408545:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205194.408548:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205194.408552:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.408555:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205194.408557:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.408560:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205194.472471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205194.472472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205194.472479:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205194.472480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205194.472482:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205194.472483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205194.472488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205194.472489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205194.472494:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205194.472494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205194.472499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.472500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205194.472504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205194.472507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205194.472511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.472511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205194.472515:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.472517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205194.664549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205194.664555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205194.664563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205194.664565:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205194.664569:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205194.664570:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205194.664580:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205194.664581:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205194.664837:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205194.664841:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205194.664843:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205194.984464:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301321729, at 4301322240 00000100:00000010:0.0:1788205194.984478:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9180. 00000100:00000001:3.0:1788205194.986027:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205194.986030:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205194.986033:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205194.986037:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205194.986040:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205194.986043:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.986045:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205194.986047:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205194.986050:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205194.986506:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301321729, at 4301322242 00000100:00000010:1.0:1788205194.986513:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9d20. 00000100:00000001:3.0:1788205195.432491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205195.432498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205195.432506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205195.432511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205195.432514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205195.432518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205195.432520:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205195.432523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205195.432525:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205195.496506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205195.496513:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205195.496516:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205195.496521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205195.496523:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205195.496527:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205195.496529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205195.496532:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205195.496534:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205195.496569:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205195.496574:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205195.496575:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205195.496579:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205195.496581:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205195.496584:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205195.496586:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205195.496588:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205195.496590:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.497298:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205195.497304:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205195.497306:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205195.497309:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205195.497312:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205195.497319:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6650 00000400:00000200:0.0:1788205195.497327:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205195.497333:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205195.497337:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205195.497340:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497342:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.497345:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497347:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205195.497351:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205195.497353:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205195.497356:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205195.497359:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205195.497361:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205195.497389:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205195.497392:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497394:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.497397:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497399:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205195.497413:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205195.497541:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497544:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205195.497547:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205195.497550:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205195.497552:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.497554:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205195.497557:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205195.497567:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000020:00000040:0.0:1788205195.497570:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205195.497574:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205195.497575:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.497578:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205195.497581:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205195.497584:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205195.497593:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb400. 00000100:00000001:0.0:1788205195.497600:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205195.497608:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205195.497617:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205195.497621:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205195.497625:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205195.497649:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.497652:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205195.497655:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205195.497656:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205195.497657:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205195.497660:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205195.497662:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6650 00000400:00000200:0.0:1788205195.497666:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205195.497669:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205195.497672:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205195.497676:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497678:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.497680:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497682:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205195.497683:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205195.497684:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205195.497688:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000001:2.0:1788205195.497689:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788205195.497690:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788205195.497694:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1788205195.497695:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205195.497699:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205195.497702:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205195.497704:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205195.497707:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497712:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.497716:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497721:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205195.497723:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788205195.497725:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:2.0:1788205195.497726:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.497730:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:2.0:1788205195.497733:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788205195.497735:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.497737:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205195.497738:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205195.497743:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205195.497744:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205195.497749:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205195.497750:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205938304:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205195.497751:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205195.497757:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788205195.497758:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205195.497762:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.497762:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205195.497767:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.497767:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.497773:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1788205195.497774:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205195.497778:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000010:2.0:1788205195.497780:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:0.0:1788205195.497781:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205195.497785:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.497785:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.497790:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205195.497792:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205195.497794:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.497797:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:0.0:1788205195.497801:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9c00. 00000400:00000010:2.0:1788205195.497804:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000001:0.0:1788205195.497806:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205195.497813:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205195.497820:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205195.497822:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205195.497823:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000040:0.0:1788205195.497828:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205195.497832:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938304, portal 25 00000100:00000001:2.0:1788205195.497838:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205195.497840:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000001:0.0:1788205195.497843:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.497847:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1788205195.497852:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205195.497853:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205195.497855:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205195.497858:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205195.497864:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788205195.497869:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788205195.497870:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6650 00000100:00000040:2.0:1788205195.497874:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205195.497878:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205195.497882:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205938304, offset 0 00000400:00000200:0.0:1788205195.497885:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.497889:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205195.497892:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205195.497896:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1788205195.497900:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788205195.497901:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497906:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788205195.497908:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:0.0:1788205195.497912:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:2.0:1788205195.497915:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205195.497917:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788205195.497920:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000020:00000001:0.0:1788205195.497924:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1788205195.497926:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1788205195.497927:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:2.0:1788205195.497929:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000001:0.0:1788205195.497932:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1788205195.497934:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59186:59186:256:4294967295] 192.168.201.151@tcp LPNI seq info [59166:59166:8:4294967295] 00000020:00000040:0.0:1788205195.497936:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:2.0:1788205195.497939:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:0.0:1788205195.497943:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:0.0:1788205195.497948:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205195.497953:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497955:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.497958:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497962:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:2.0:1788205195.497963:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000020:00000040:0.0:1788205195.497966:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205195.497974:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205195.497977:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788205195.497979:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788205195.497980:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205195.497985:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1788205195.497987:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 02000000:00000001:0.0:1788205195.497988:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.497993:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205195.497996:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000800:00000200:2.0:1788205195.497999:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000008:00000001:0.0:1788205195.498001:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498005:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788205195.498006:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000100:00000010:0.0:1788205195.498010:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000800:00000200:2.1:1788205195.498016:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000040:0.0:1788205195.498017:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000200:2.1:1788205195.498020:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 02000000:00000001:0.0:1788205195.498022:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205195.498026:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498030:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:2.0:1788205195.498030:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788205195.498034:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:2.0:1788205195.498035:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 02000000:00000001:0.0:1788205195.498037:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.498043:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:0.0:1788205195.498048:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcac00. 00000400:00000200:2.0:1788205195.498050:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788205195.498053:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.498057:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000100:00100000:0.0:1788205195.498062:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1788205195.498063:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205195.498070:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788205195.498074:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1788205195.498077:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205195.498079:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205195.498088:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498093:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.498100:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205195.498101:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.498107:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205195.498117:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205195.498118:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498122:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788205195.498123:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788205195.498127:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205195.498131:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1788205195.498131:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205195.498138:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6650 00000400:00000200:2.0:1788205195.498139:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205195.498143:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301326850 00000400:00000200:0.0:1788205195.498158:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:2.0:1788205195.498160:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1660. 00000400:00000200:0.0:1788205195.498165:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205195.498168:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788205195.498169:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1788205195.498174:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.498178:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788205195.498178:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301326850 00000100:00000040:0.0:1788205195.498182:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:2.0:1788205195.498184:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1600. 00000100:00000001:0.0:1788205195.498188:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205195.498190:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788205195.498193:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205195.498195:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:2.0:1788205195.498195:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788205195.498199:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788205195.498201:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205195.498203:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:2.0:1788205195.498206:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788205195.498207:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:0.0:1788205195.498211:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00100000:2.0:1788205195.498215:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205195/real 1788205195] req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205195.498218:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.498223:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205195.498229:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205195.498233:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000001:2.0:1788205195.498234:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1788205195.498236:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000001:2.0:1788205195.498238:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205195.498241:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000100:00080000:0.0:1788205195.498242:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788205195.498244:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000100:00000001:0.0:1788205195.498247:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205195.498247:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 02000000:00000001:0.0:1788205195.498253:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788205195.498253:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1788205195.498259:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1788205195.498259:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205195.498263:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498265:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205195.498267:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205195.498272:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498274:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205195.498282:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000100:00000200:2.0:1788205195.498286:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205195.498287:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:0.0:1788205195.498293:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205195.498295:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205195.498300:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205195.498303:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788205195.498303:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205195.498307:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205195.498308:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498314:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:0.0:1788205195.498316:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dcb000. 00000100:00000001:2.0:1788205195.498318:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205195.498322:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205195.498324:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205195.498331:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205195.498337:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205195.498346:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1788205195.498347:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205195.498351:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498359:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1788205195.498361:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498362:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498382:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.498383:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498385:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205195.498388:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205195.498392:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205195.498399:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205195.498403:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498403:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205195.498411:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6660) 00000100:00000040:2.0:1788205195.498411:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205195.498419:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205938304:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205195.498423:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498424:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.498427:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205938304/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498431:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205195.498433:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205195.498434:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205195.498437:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205195.498439:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205195.498440:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788205195.498441:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205195.498443:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb400. 02000000:00000001:2.0:1788205195.498445:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205195.498447:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205195.498448:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:2.0:1788205195.498449:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205195.498450:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498452:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498457:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205195.498458:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498466:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205195.498467:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205195.498469:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205195.498472:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498481:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205195.498482:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498485:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205195.498486:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205195.498488:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205195.498491:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938432:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205195.498494:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205195.498496:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205195.498497:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498499:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205195.498502:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205195.498504:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.498508:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205195.498510:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205195.498514:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205195.498517:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938432, portal 10 00000100:00000001:2.0:1788205195.498518:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205195.498519:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:2.0:1788205195.498524:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498528:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205195.498529:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205195.498531:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205195.498532:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205938432, offset 0 00000400:00000200:2.0:1788205195.498534:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205195.498537:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.498539:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205195.498541:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205195.498543:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205195.498544:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205195.498545:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205195.498547:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59187:59187:256:4294967295] 192.168.201.151@tcp LPNI seq info [59167:59167:8:4294967295] 00000400:00000200:2.0:1788205195.498550:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.498554:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205195.498556:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.498558:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205195.498561:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205195.498563:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788205195.498566:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205195.498568:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788205195.498570:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.498572:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205195.498575:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205195.498578:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205195.498581:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205195.498582:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205195.498585:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205195.498588:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498595:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.498598:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498605:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498606:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205195.498610:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205195.498613:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788205195.498616:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205195.498618:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301322908 00000100:00000001:2.0:1788205195.498620:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.498622:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301322908 00000100:00000001:2.0:1788205195.498623:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.498624:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205195.498626:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498627:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205195.498629:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205195/real 1788205195] req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205195.498638:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205195.498639:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205195.498640:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205195.498642:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205195.498643:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205195.498644:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205195.498646:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205195.498651:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498654:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205195.498655:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498656:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.498657:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205195.498659:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498662:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205195.498665:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498669:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205195.498670:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498671:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.498672:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498673:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205195.498674:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205195.498676:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205195.498677:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205195.498679:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205195.498681:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205195.498686:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938432:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205195.498688:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205195.498690:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498693:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205195.498694:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498695:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205195.498697:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205195.498698:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205195.498699:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938560:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205195.498701:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205195.498702:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205195.498703:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498704:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205195.498707:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205195.498708:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.498710:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205195.498711:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205195.498713:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205195.498715:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938560, portal 4 00000100:00000001:2.0:1788205195.498717:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205195.498718:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:2.0:1788205195.498724:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498730:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205195.498732:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205195.498734:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205195.498736:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205938560, offset 0 00000400:00000200:2.0:1788205195.498739:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205195.498743:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.498746:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205195.498749:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205195.498751:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205195.498754:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205195.498756:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205195.498758:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59188:59188:256:4294967295] 192.168.201.151@tcp LPNI seq info [59168:59168:8:4294967295] 00000400:00000200:2.0:1788205195.498762:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.498768:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205195.498772:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.498776:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205195.498780:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205195.498783:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788205195.498787:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205195.498790:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788205195.498793:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.498795:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205195.498798:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205195.498801:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205195.498804:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205195.498805:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205195.498808:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205195.498811:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498818:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.498821:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498827:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498829:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205195.498832:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205195.498835:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788205195.498838:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205195.498840:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301322794 00000100:00000001:2.0:1788205195.498842:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.498843:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301322794 00000100:00000001:2.0:1788205195.498845:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.498847:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205195.498849:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.498851:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205195.498855:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205195/real 1788205195] req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205195.498862:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205195.498864:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205195.498865:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205195.498867:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205195.498869:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205195.498870:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205195.498873:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205195.498880:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498886:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205195.498887:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498889:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.498890:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205195.498893:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498898:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205195.498902:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498908:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205195.498911:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498912:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.498914:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.498915:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205195.498917:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205195.498919:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205195.498922:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205195.498924:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205195.498927:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205195.498934:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938560:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205195.498938:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205195.498940:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498945:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205195.498945:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498947:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205195.498948:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205195.498950:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205195.498952:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205195.498955:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205195.498957:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205195.498959:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205195.498960:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205195.498964:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205195.498966:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205195.498969:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205195.498971:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205195.498975:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205195.498978:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938688, portal 4 00000100:00000001:2.0:1788205195.498980:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205195.498981:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000100:00000040:2.0:1788205195.498987:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.498992:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205195.498994:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205195.498997:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205195.498999:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205938688, offset 0 00000400:00000200:2.0:1788205195.499002:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205195.499006:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.499010:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205195.499013:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205195.499015:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205195.499017:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205195.499019:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205195.499022:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59189:59189:256:4294967295] 192.168.201.151@tcp LPNI seq info [59169:59169:8:4294967295] 00000400:00000200:2.0:1788205195.499026:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205195.499032:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205195.499035:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.499037:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205195.499040:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205195.499041:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1788205195.499044:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205195.499046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1788205195.499048:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205195.499049:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205195.499051:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205195.499053:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205195.499055:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205195.499056:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205195.499058:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205195.499060:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499069:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.499071:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499075:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.499076:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205195.499077:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205195.499079:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788205195.499081:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205195.499082:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301322800 00000100:00000001:2.0:1788205195.499083:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.499085:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301322800 00000100:00000001:2.0:1788205195.499086:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301326850 00000100:00000001:2.0:1788205195.499087:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205195.499088:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.499089:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205195.499091:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205195/real 1788205195] req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205195.499095:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205195.499096:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205195.499097:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205195.499098:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205195.499099:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205195.499100:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205195.499102:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205195.499107:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499110:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205195.499111:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499112:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.499113:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205195.499115:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499118:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205195.499120:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499124:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205195.499125:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499126:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205195.499126:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499127:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205195.499129:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205195.499130:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205195.499132:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205195.499133:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205195.499135:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205195.499139:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938688:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205195.499140:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499142:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.499143:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205938432/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499161:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205195.499163:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205195.499165:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205195.499167:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205195.499169:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205195.499170:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788205195.499172:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205195.499173:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9c00. 02000000:00000001:2.0:1788205195.499174:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205195.499175:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205195.499176:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:2.0:1788205195.499178:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205195.499179:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499180:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.499182:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205938560/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499186:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205195.499187:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205195.499188:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205195.499189:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205195.499190:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205195.499190:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788205195.499191:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205195.499192:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcac00. 02000000:00000001:2.0:1788205195.499193:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205195.499194:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205195.499194:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:2.0:1788205195.499196:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205195.499196:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499197:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205195.499199:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205938688/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205195.499202:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205195.499202:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205195.499203:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205195.499204:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205195.499205:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205195.499205:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788205195.499206:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205195.499207:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dcb000. 02000000:00000001:2.0:1788205195.499208:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205195.499209:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205195.499210:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:2.0:1788205195.499211:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205195.499212:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499213:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.499216:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205195.499216:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.499217:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205195.499218:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205195.688510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205195.688516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205195.688521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205195.688522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205195.688526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205195.688527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205195.688529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205195.688531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205195.688671:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205195.688675:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205195.688679:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205196.008516:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205196.008526:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205196.008528:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205196.008535:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205196.008539:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205196.008544:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.008547:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205196.009207:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.009214:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205196.456438:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205196.456443:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205196.456445:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205196.456448:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205196.456452:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205196.456456:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.456458:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205196.456460:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.456463:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205196.522241:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205196.522247:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205196.522256:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205196.522261:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205196.522264:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205196.522268:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.522271:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205196.522273:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.522277:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205196.522392:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205196.522395:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205196.522396:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205196.522399:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205196.522401:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205196.522403:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205196.522405:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205196.522406:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205196.522408:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205196.713783:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205196.713789:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205196.713794:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205196.713796:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205196.713799:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205196.713800:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205196.713804:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205196.713805:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205196.715254:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205196.715259:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205196.715318:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205197.032668:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205197.032675:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205197.032677:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.032682:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205197.032696:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205197.032700:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.032703:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205197.032705:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.032708:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.480528:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205197.480535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205197.480537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.480542:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205197.480546:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205197.480549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.480552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205197.480554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.480556:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.544948:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205197.544956:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205197.544957:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.544963:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205197.544966:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205197.544969:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.544972:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.544974:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.544976:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.545075:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205197.545078:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205197.545079:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.545082:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205197.545084:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205197.545086:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.545088:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205197.545089:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.545091:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205197.738159:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205197.738163:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205197.738167:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205197.738168:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205197.738170:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205197.738171:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205197.738173:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205197.738174:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205197.738234:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205197.738238:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205197.738241:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205198.056913:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205198.056919:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205198.056921:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205198.056927:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205198.056930:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205198.056934:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.056937:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205198.056940:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.056943:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205198.504980:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205198.504987:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205198.504990:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205198.504996:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205198.504999:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205198.505003:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.505006:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205198.505008:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.505011:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205198.568429:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205198.568438:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205198.568440:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205198.568446:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205198.568449:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205198.568453:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.568456:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205198.568470:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205198.568474:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205198.568541:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205198.568544:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205198.568546:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205198.568549:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205198.568551:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205198.568553:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205198.568554:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205198.568556:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205198.568558:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205198.760601:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205198.760608:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205198.760614:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205198.760616:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205198.760619:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205198.760621:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205198.760623:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205198.760625:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205198.760688:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205198.760692:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205198.760695:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205199.080613:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205199.080620:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205199.080622:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205199.080627:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205199.080630:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205199.080634:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.080637:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205199.080639:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.080642:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205199.529437:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205199.529444:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205199.529446:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205199.529451:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205199.529455:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205199.529458:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.529460:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205199.529462:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.529465:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205199.592471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205199.592471:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205199.592480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205199.592480:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205199.592483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205199.592483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205199.592490:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205199.592490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205199.592495:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205199.592496:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205199.592499:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.592500:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.592505:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205199.592507:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.592510:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205199.592512:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205199.592515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205199.592518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205199.784474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205199.784479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205199.784484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205199.784485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205199.784488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205199.784489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205199.784492:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205199.784494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205199.784552:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205199.784556:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205199.784560:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205200.104471:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301326850, at 4301327360 00000100:00000001:3.0:1788205200.104480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:1.0:1788205200.104484:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1600. 00000100:00000001:3.0:1788205200.104486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205200.104489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205200.104494:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205200.104497:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205200.104501:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.104503:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205200.104506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.104509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205200.104554:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301326850, at 4301327360 00000100:00000010:0.0:1788205200.104561:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1660. 00000100:00000001:3.0:1788205200.552451:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205200.552458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205200.552461:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205200.552466:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205200.552469:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205200.552473:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.552475:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205200.552477:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.552480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205200.616465:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1788205200.616465:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205200.616467:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205200.616472:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205200.616473:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205200.616476:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205200.616476:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205200.616477:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205200.616481:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205200.616483:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205200.616483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205200.616485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205200.616489:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205200.616489:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205200.616490:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.616494:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205200.616496:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205200.616497:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205200.616501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205200.616506:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6655 00000100:00000001:0.0:1788205200.616507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205200.616507:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205200.616512:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205200.616516:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205200.616520:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205200.616527:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205200.616530:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205200.616534:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616537:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205200.616541:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616543:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205200.616547:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205200.616550:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205200.616552:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205200.616555:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205200.616557:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205200.616560:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205200.616563:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616564:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205200.616566:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616568:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205200.616571:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205200.616573:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616576:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205200.616578:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205200.616581:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205200.616584:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.616586:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205200.616588:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205200.616596:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788205200.616599:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205200.616603:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205200.616605:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.616607:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205200.616609:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205200.616613:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205200.616620:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000001:1.0:1788205200.616633:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205200.616642:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205200.616651:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205200.616654:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205200.616658:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205200.616683:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205200.616687:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205200.616689:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205200.616690:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205200.616691:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205200.616693:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205200.616695:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6655 00000400:00000200:1.0:1788205200.616698:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205200.616701:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205200.616704:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205200.616707:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616710:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205200.616712:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205200.616713:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205200.616714:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205200.616717:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788205200.616719:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788205200.616720:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788205200.616724:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000100:00000040:2.0:1788205200.616728:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205200.616729:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205200.616733:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205200.616739:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:2.0:1788205200.616740:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205200.616742:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205200.616747:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:2.0:1788205200.616747:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205200.616750:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205200.616753:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205200.616756:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:2.0:1788205200.616758:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205938816:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205200.616762:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788205200.616763:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205200.616768:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1788205200.616769:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:2.0:1788205200.616770:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.616774:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788205200.616775:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205200.616779:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 00000020:00000040:1.0:1788205200.616780:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 02000000:00000001:2.0:1788205200.616782:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205200.616787:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:2.0:1788205200.616789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205200.616793:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000001:1.0:1788205200.616794:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205200.616799:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205200.616804:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205200.616807:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205200.616807:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000001:1.0:1788205200.616811:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:2.0:1788205200.616814:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938816, portal 25 00000008:00100000:1.0:1788205200.616817:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788205200.616818:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205200.616820:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000008:00000001:1.0:1788205200.616824:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205200.616827:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205200.616828:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205200.616837:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000100:00000001:2.0:1788205200.616837:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205200.616840:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:1.0:1788205200.616843:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:2.0:1788205200.616844:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205200.616849:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205938816, offset 0 02000000:00000001:1.0:1788205200.616850:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205200.616853:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.616856:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205200.616859:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205200.616862:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1788205200.616865:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:1.0:1788205200.616866:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.616871:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000010:1.0:1788205200.616873:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000400:00000200:2.0:1788205200.616877:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205200.616880:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.616881:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205200.616885:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205200.616890:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00100000:1.0:1788205200.616892:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788205200.616893:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59190:59190:256:4294967295] 192.168.201.151@tcp LPNI seq info [59170:59170:8:4294967295] 00000400:00000200:2.0:1788205200.616902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205200.616911:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1788205200.616912:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788205200.616917:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1788205200.616920:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.616925:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00000040:1.0:1788205200.616926:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788205200.616933:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205200.616938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788205200.616942:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205200.616944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1788205200.616952:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205200.616956:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205200.616963:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1788205200.616963:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.616965:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000100:00000001:1.0:1788205200.616967:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205200.616970:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788205200.616971:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205200.616977:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788205200.616978:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1788205200.616982:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205200.616984:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205200.616988:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1788205200.616990:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6655 00000100:00000200:2.0:1788205200.616994:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205200.617000:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205200.617004:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1788205200.617009:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:2.0:1788205200.617009:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205200.617014:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788205200.617018:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617021:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1788205200.617022:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.617026:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205200.617028:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.617030:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205200.617033:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788205200.617033:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205200.617038:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000200:2.0:1788205200.617040:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788205200.617042:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788205200.617044:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301331969 00000100:00000010:2.0:1788205200.617047:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1420. 00000100:00000001:2.0:1788205200.617052:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1788205200.617053:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:2.0:1788205200.617070:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301331969 00000020:00000001:1.0:1788205200.617073:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000010:2.0:1788205200.617076:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1c60. 00000020:00000001:1.0:1788205200.617078:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:2.0:1788205200.617081:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1788205200.617085:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205200.617086:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205200.617091:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788205200.617091:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617094:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788205200.617097:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205200.617101:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:2.0:1788205200.617103:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205200/real 1788205200] req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205200.617106:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205200.617114:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000001:2.0:1788205200.617115:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205200.617118:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788205200.617120:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205200.617120:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 02000000:00000001:1.0:1788205200.617126:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1788205200.617126:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205200.617128:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205200.617131:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1788205200.617132:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1788205200.617134:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205200.617136:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.617142:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205200.617147:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000200:2.0:1788205200.617147:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1788205200.617155:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.617160:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617166:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000010:1.0:1788205200.617168:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000100:00000001:2.0:1788205200.617168:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617171:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617173:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788205200.617174:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00100000:2.0:1788205200.617176:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205200.617180:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205200.617184:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617185:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205200.617190:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1788205200.617191:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205200.617193:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205200.617197:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617199:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205200.617202:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1788205200.617203:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:2.0:1788205200.617203:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617205:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617208:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205200.617210:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205200.617211:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205200.617212:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205200.617217:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205200.617219:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788205200.617222:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1788205200.617223:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205200.617248:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205938816:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205200.617252:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205200.617253:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617256:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205200.617258:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205200.617259:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205938816/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617265:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205200.617266:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205200.617268:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 00000100:00000040:1.0:1788205200.617271:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205200.617272:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205200.617275:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205200.617277:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788205200.617279:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205200.617282:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:2.0:1788205200.617286:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205200.617288:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205200.617290:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:1.0:1788205200.617291:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617292:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205200.617295:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205200.617297:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788205200.617297:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205200.617303:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205200.617304:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205200.617306:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.617308:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788205200.617309:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205200.617312:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788205200.617313:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205200.617313:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205200.617319:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205200.617320:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205200.617326:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6655 00000100:00000001:2.0:1788205200.617332:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1788205200.617334:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205200.617336:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205200.617340:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1788205200.617341:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 02000000:00000001:2.0:1788205200.617343:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788205200.617346:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788205200.617348:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205200.617353:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:2.0:1788205200.617355:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938944:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205200.617357:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788205200.617362:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1788205200.617394:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:2.0:1788205200.617396:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788205200.617399:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:2.0:1788205200.617400:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205200.617404:0:30130:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788205200.617405:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1788205200.617409:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 02000000:00000010:2.0:1788205200.617410:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 00000020:00000001:1.0:1788205200.617413:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788205200.617415:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205200.617418:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000400:00000200:2.0:1788205200.617422:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000001:1.0:1788205200.617424:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205200.617428:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205200.617433:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:2.0:1788205200.617434:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000001:1.0:1788205200.617436:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1788205200.617439:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000040:1.0:1788205200.617442:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:2.0:1788205200.617444:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205938944, portal 10 00000100:00000001:1.0:1788205200.617447:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205200.617449:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1788205200.617453:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:2.0:1788205200.617453:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00080000:1.0:1788205200.617460:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:2.0:1788205200.617464:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205200.617467:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205200.617472:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788205200.617473:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205200.617477:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788205200.617478:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1788205200.617479:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205200.617482:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205938944, offset 0 02000000:00000001:1.0:1788205200.617484:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617487:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205200.617490:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:2.0:1788205200.617493:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00100000:1.0:1788205200.617495:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:2.0:1788205200.617497:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000008:00000001:1.0:1788205200.617501:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617501:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205200.617506:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617506:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205200.617511:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000010:1.0:1788205200.617514:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000400:00000200:2.0:1788205200.617515:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000020:00000040:1.0:1788205200.617520:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:2.0:1788205200.617521:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59191:59191:256:4294967295] 192.168.201.151@tcp LPNI seq info [59171:59171:8:4294967295] 02000000:00000001:1.0:1788205200.617526:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205200.617530:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617530:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205200.617535:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205200.617539:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205200.617543:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617543:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 02000000:00000010:1.0:1788205200.617549:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000800:00000200:2.0:1788205200.617551:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205200.617555:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1788205200.617557:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000100:00100000:1.0:1788205200.617565:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1788205200.617565:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205200.617571:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788205200.617582:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205200.617583:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1788205200.617587:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1788205200.617588:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788205200.617594:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.617598:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000100:00000040:1.0:1788205200.617599:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205200.617602:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205200.617610:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205200.617616:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205200.617620:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:1.0:1788205200.617621:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617623:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1788205200.617629:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6665) 00000100:00000200:2.0:1788205200.617629:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617646:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.617648:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617653:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617654:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205200.617656:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205200.617659:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788205200.617661:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205200.617662:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301328027 00000100:00000001:2.0:1788205200.617664:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.617666:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301328027 00000100:00000001:2.0:1788205200.617667:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.617668:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205200.617669:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617670:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205200.617672:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205200/real 1788205200] req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205200.617676:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205200.617677:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205200.617678:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205200.617680:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205200.617681:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205200.617682:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205200.617684:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205200.617688:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617694:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205200.617696:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617698:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617699:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205200.617702:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617709:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205200.617712:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617719:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205200.617722:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617723:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617725:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617726:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205200.617729:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205200.617731:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205200.617734:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205200.617737:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205200.617741:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205200.617749:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205938944:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205200.617752:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205200.617755:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617761:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205200.617763:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205200.617765:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205200.617767:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205200.617769:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205200.617772:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939072:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205200.617775:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205200.617776:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205200.617778:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205200.617779:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205200.617782:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205200.617784:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.617788:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205200.617790:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205200.617793:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205200.617795:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939072, portal 4 00000100:00000001:2.0:1788205200.617797:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205200.617799:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:2.0:1788205200.617803:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617810:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205200.617812:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205200.617814:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205200.617816:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205939072, offset 0 00000400:00000200:2.0:1788205200.617819:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205200.617822:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205200.617825:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205200.617828:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205200.617830:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205200.617832:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205200.617833:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205200.617835:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59192:59192:256:4294967295] 192.168.201.151@tcp LPNI seq info [59172:59172:8:4294967295] 00000400:00000200:2.0:1788205200.617837:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205200.617841:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205200.617844:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.617846:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205200.617848:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205200.617850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788205200.617853:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205200.617854:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788205200.617857:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.617858:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205200.617860:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205200.617862:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205200.617864:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205200.617865:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205200.617866:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205200.617868:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617873:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.617875:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617878:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617879:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205200.617881:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205200.617883:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788205200.617884:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205200.617885:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301327913 00000100:00000001:2.0:1788205200.617887:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.617888:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301327913 00000100:00000001:2.0:1788205200.617889:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.617891:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205200.617892:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.617893:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205200.617895:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205200/real 1788205200] req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205200.617899:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205200.617900:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205200.617901:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205200.617902:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205200.617903:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205200.617904:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205200.617906:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205200.617910:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617913:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205200.617914:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617915:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617916:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205200.617919:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617925:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205200.617929:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617936:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205200.617938:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617939:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.617940:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617942:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205200.617944:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205200.617946:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205200.617950:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205200.617952:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205200.617956:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205200.617969:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939072:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205200.617973:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.617975:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.617978:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205938944/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.617984:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205200.617985:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205200.617987:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205200.617990:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205200.617992:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205200.617993:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788205200.617995:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205200.617997:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:2.0:1788205200.617999:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205200.618000:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205200.618001:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:2.0:1788205200.618004:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205200.618005:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618006:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.618009:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205939072/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618015:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205200.618016:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205200.618017:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205200.618019:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205200.618020:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205200.618021:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788205200.618023:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205200.618024:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:2.0:1788205200.618026:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205200.618026:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205200.618027:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:2.0:1788205200.618029:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205200.618030:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618031:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618034:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205200.618034:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.618036:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205200.618037:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205200.618038:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205200.618040:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618044:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205200.618045:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205200.618046:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205200.618047:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205200.618048:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205200.618050:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205200.618052:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205200.618053:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205200.618054:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205200.618055:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205200.618057:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205200.618070:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205200.618073:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205200.618074:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205200.618076:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205200.618078:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939200, portal 4 00000100:00000001:2.0:1788205200.618080:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205200.618080:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 00000100:00000040:2.0:1788205200.618084:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618088:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205200.618089:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205200.618090:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205200.618092:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205939200, offset 0 00000400:00000200:2.0:1788205200.618094:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205200.618097:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205200.618099:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205200.618101:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205200.618102:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205200.618104:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205200.618105:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205200.618107:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59193:59193:256:4294967295] 192.168.201.151@tcp LPNI seq info [59173:59173:8:4294967295] 00000400:00000200:2.0:1788205200.618110:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205200.618113:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205200.618115:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.618117:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205200.618119:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205200.618121:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1788205200.618124:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205200.618125:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1788205200.618128:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205200.618129:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205200.618130:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205200.618132:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205200.618134:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205200.618135:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205200.618137:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205200.618139:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618143:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.618145:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618149:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.618150:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205200.618151:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205200.618153:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788205200.618155:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205200.618156:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301327919 00000100:00000001:2.0:1788205200.618158:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.618159:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301327919 00000100:00000001:2.0:1788205200.618160:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301331969 00000100:00000001:2.0:1788205200.618161:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205200.618162:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.618163:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205200.618165:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205200/real 1788205200] req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205200.618169:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205200.618170:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205200.618171:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205200.618172:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205200.618173:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205200.618174:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205200.618176:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205200.618180:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618184:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205200.618185:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618186:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.618187:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205200.618189:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618192:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205200.618194:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618197:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205200.618199:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618200:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205200.618201:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618202:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205200.618203:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205200.618205:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205200.618206:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205200.618207:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205200.618209:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205200.618213:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939200:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205200.618215:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618217:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205200.618218:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205939200/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205200.618222:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205200.618223:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205200.618224:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:2.0:1788205200.618225:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205200.618226:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205200.618226:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788205200.618228:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205200.618229:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:2.0:1788205200.618230:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205200.618231:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205200.618232:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:2.0:1788205200.618233:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205200.618233:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618235:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205200.618237:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205200.618238:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205200.618240:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205200.618242:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205200.808497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205200.808504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205200.808510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205200.808512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205200.808515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205200.808516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205200.808519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205200.808520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205200.808579:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205200.808583:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205200.808586:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205201.128526:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205201.128535:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205201.128537:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205201.128542:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205201.128546:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205201.128550:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.128554:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205201.128557:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.128560:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205201.576498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205201.576505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205201.576507:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205201.576513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205201.576516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205201.576519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.576522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205201.576524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.576527:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205201.640475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205201.640476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205201.640484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205201.640485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205201.640488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205201.640494:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205201.640498:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205201.640501:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.640501:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205201.640507:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205201.640508:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205201.640512:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.640513:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205201.640518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205201.640518:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.640524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205201.640526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.640529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832465:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205201.832472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205201.832480:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205201.832482:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205201.832485:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832487:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205201.832489:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832491:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832494:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205201.832498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205201.832501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205201.832502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205201.832556:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205201.832560:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205201.832563:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205202.152475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205202.152494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205202.152497:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205202.152502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205202.152505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205202.152509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.152511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205202.152515:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.152517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205202.600527:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205202.600535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205202.600537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205202.600542:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205202.600546:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205202.600550:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.600552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205202.600554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.600557:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205202.664483:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205202.664483:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205202.664490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205202.664491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205202.664495:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205202.664496:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205202.664502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205202.664503:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205202.664507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205202.664508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205202.664512:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.664513:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205202.664518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205202.664521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205202.664525:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205202.664525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205202.664529:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.664533:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205202.856484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205202.856490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205202.856493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205202.856494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205202.856496:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205202.856496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205202.856498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205202.856499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205202.856569:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205202.856572:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205202.856574:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205203.176454:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205203.176461:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205203.176462:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205203.176467:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205203.176471:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205203.176474:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.176477:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205203.176479:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.176482:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205203.624502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205203.624510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205203.624512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205203.624517:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205203.624521:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205203.624525:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.624527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205203.624542:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.624546:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205203.688480:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205203.688480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205203.688489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205203.688489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205203.688494:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205203.688494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205203.688502:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205203.688502:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205203.688508:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205203.688509:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205203.688512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205203.688513:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.688517:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205203.688521:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205203.688524:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205203.688525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205203.688528:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205203.688531:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205203.880520:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205203.880523:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205203.880525:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880530:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205203.880532:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880534:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880536:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205203.880541:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205203.880547:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205203.880548:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205203.880549:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880551:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205203.880552:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880554:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880555:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205203.880561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205203.880562:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205203.880563:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880565:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205203.880566:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880568:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205203.880569:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205203.882241:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205203.882245:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205203.882248:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205204.200514:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205204.200520:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205204.200522:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205204.200525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205204.200528:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205204.200531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.200533:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205204.200535:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.200537:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205204.648498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205204.648505:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205204.648508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205204.648513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205204.648516:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205204.648520:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.648522:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205204.648525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.648528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205204.712473:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205204.712473:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205204.712480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205204.712481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205204.712483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205204.712484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205204.712488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205204.712490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205204.712492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205204.712495:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205204.712497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.712499:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205204.712500:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205204.712504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205204.712507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205204.712511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205204.712514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.712517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205204.904507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205204.904514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205204.904519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205204.904520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205204.904523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205204.904524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205204.904525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205204.904526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205204.904561:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205204.904564:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205204.904567:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205205.224489:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301331969, at 4301332480 00000100:00000001:3.0:1788205205.224489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788205205.224491:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301331969, at 4301332480 00000100:00000001:3.0:1788205205.224497:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205205.224504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205205.224511:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:0.0:1788205205.224512:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1420. 00000100:00000010:1.0:1788205205.224514:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1c60. 00000100:00000001:3.0:1788205205.224518:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205205.224524:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.224527:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205205.224530:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.224532:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205205.672502:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205205.672510:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205205.672512:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205205.672518:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205205.672521:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205205.672525:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.672527:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205205.672530:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.672532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205205.736501:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205205.736503:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205205.736516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1788205205.736517:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205205.736522:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205205.736525:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.736526:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205205.736529:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205205.736533:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205205.736535:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788205205.736536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205205.736538:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205205.736544:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205205.736546:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205205.736548:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205205.736550:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.736551:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205205.736556:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788205205.736558:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6660 00000100:00000001:0.0:1788205205.736562:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205205.736569:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205205.736582:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205205.736583:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205205.736590:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205205.736590:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205205.736600:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205205.736601:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205205.736611:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736616:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.736620:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736624:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205205.736628:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205205.736631:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205205.736634:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205205.736637:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205205.736639:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205205.736640:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205205.736643:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736644:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.736645:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736646:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205205.736648:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205205.736650:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736651:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205205.736653:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205205.736656:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205205.736658:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.736659:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205205.736661:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205205.736669:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788205205.736672:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205205.736675:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205205.736676:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.736678:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205205.736680:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205205.736683:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205205.736689:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788205205.736694:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205205.736702:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205205.736717:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205205.736721:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205205.736725:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205205.736747:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205205.736750:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205205.736752:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205205.736753:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205205.736754:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205205.736756:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205205.736758:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6660 00000400:00000200:1.0:1788205205.736761:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205205.736763:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205205.736765:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205205.736767:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736768:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.736770:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736772:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205205.736773:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205205.736775:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205205.736777:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205205.736779:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205205.736780:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205205.736781:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788205205.736783:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736784:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.736786:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736787:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205205.736789:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205205.736790:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736792:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205205.736793:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205205.736797:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205205.736798:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.736802:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205205.736802:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1788205205.736809:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788205205.736811:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:1.0:1788205205.736812:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.736817:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205205.736818:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205205.736829:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788205205.736832:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205205.736835:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205205.736836:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205205.736838:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205205.736839:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205205.736845:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205205.736848:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205205.736856:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000100:00000001:1.0:1788205205.736864:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.736866:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205205.736869:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205205.736871:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788205205.736876:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205205.736878:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205205.736883:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.736884:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205205.736889:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205205.736892:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205205.736906:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205939328:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205205.736910:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.736915:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1788205205.736916:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205205.736921:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205205.736922:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1788205205.736923:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788205205.736924:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205205.736928:0:30130:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1788205205.736929:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205205.736933:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6660 00000400:00000200:1.0:1788205205.736941:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205205.736942:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1788205205.736947:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205205.736951:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 02000000:00000010:2.0:1788205205.736952:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205205.736960:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205205.736961:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.736967:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1788205205.736984:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205205.736990:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000100:00000040:1.0:1788205205.736999:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.737001:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205205.737004:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205205.737005:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000400:00000010:2.0:1788205205.737005:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000020:00000001:1.0:1788205205.737011:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205205.737014:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205205.737017:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205205.737018:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205205.737022:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000200:2.0:1788205205.737022:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939328, portal 25 00000100:00000001:1.0:1788205205.737031:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.737034:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205205.737034:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205205.737039:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:2.0:1788205205.737039:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000020:00000040:1.0:1788205205.737051:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205205.737062:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000040:2.0:1788205205.737063:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205205.737064:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205205.737070:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205205.737077:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205205.737086:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.737094:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205205.737094:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:1.0:1788205205.737104:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000040:2.0:1788205205.737113:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000008:00000001:1.0:1788205205.737114:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.737122:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205205.737123:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000010:1.0:1788205205.737126:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788205205.737129:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000200:2.0:1788205205.737130:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205939328, offset 0 02000000:00000001:1.0:1788205205.737132:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205205.737135:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.737138:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205205.737140:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1788205205.737145:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788205205.737147:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205205.737154:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000400:00000200:2.0:1788205205.737158:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205205.737159:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205205.737171:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1788205205.737171:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205205.737184:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205205.737185:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205205.737191:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205205.737196:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:1.0:1788205205.737197:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205205.737210:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205205.737210:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1788205205.737218:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1788205205.737222:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00080000:1.0:1788205205.737223:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205205.737228:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205205.737234:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1788205205.737236:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59194:59194:256:4294967295] 192.168.201.151@tcp LPNI seq info [59174:59174:8:4294967295] 00000100:00000001:1.0:1788205205.737238:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205205.737244:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6660 00000400:00000200:1.0:1788205205.737260:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.737260:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205205.737270:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205205.737273:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205205.737284:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:2.0:1788205205.737285:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:1.0:1788205205.737292:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205205.737301:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.0:1788205205.737302:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205205.737305:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205205.737308:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205205.737309:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205205.737311:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:2.0:1788205205.737311:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000020:00000040:1.0:1788205205.737315:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205205.737319:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205205.737321:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:2.0:1788205205.737328:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:1.0:1788205205.737330:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205205.737340:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1788205205.737341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000100:00000040:1.0:1788205205.737353:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:2.1:1788205205.737354:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205205.737362:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:2.1:1788205205.737400:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000020:00000040:1.0:1788205205.737401:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205205.737411:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205205.737416:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788205205.737424:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 02000000:00000001:1.0:1788205205.737426:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:2.0:1788205205.737429:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 02000000:00000001:1.0:1788205205.737432:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205205.737435:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.737437:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205205.737439:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000400:00000200:2.0:1788205205.737440:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:1.0:1788205205.737441:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205205.737449:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205205.737454:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000400:00000200:2.0:1788205205.737455:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000020:00000040:1.0:1788205205.737460:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:2.0:1788205205.737464:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 02000000:00000001:1.0:1788205205.737465:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205205.737470:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1788205205.737472:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:1.0:1788205205.737473:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205205.737478:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205205.737481:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.737484:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:1.0:1788205205.737485:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788205205.737490:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1788205205.737494:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205205.737499:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205205.737547:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205205.737548:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205205.737553:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205205.737561:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205205.737562:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737610:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.737613:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205205.737614:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.737623:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205205.737624:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6670) 00000400:00000200:2.0:1788205205.737636:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205205.737641:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301337090 00000100:00000010:2.0:1788205205.737647:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c17e0. 00000100:00000001:2.0:1788205205.737656:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205205.737663:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301337090 00000100:00000010:2.0:1788205205.737666:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3103c1cc0. 00000100:00000001:2.0:1788205205.737670:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788205205.737674:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205205.737677:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.737679:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205205.737684:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205205/real 1788205205] req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205205.737694:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205205.737696:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205205.737698:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205205.737701:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205205.737702:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205205.737705:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205205.737708:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205205.737717:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737724:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205205.737726:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737728:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.737730:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205205.737735:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737742:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205205.737747:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737761:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205205.737764:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737766:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.737767:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737770:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205205.737772:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205205.737775:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205205.737780:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205205.737783:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205205.737787:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205205.737796:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205939328:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205205.737800:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737802:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.737805:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205939328/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737812:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205205.737815:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205205.737818:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205205.737821:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205205.737823:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205205.737825:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:2.0:1788205205.737827:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205205.737830:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:2.0:1788205205.737833:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205205.737835:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205205.737837:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:2.0:1788205205.737841:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205205.737842:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737844:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.737852:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205205.737854:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.737856:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205205.737858:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205205.737859:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205205.737863:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737870:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205205.737872:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.737874:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205205.737876:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205205.737878:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205205.737881:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939456:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205205.737884:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205205.737886:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205205.737888:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.737890:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205205.737893:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205205.737895:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205205.737898:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205205.737901:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205205.737904:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205205.737907:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939456, portal 10 00000100:00000001:2.0:1788205205.737909:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205205.737910:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:2.0:1788205205.737916:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.737923:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205205.737925:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205205.737927:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205205.737930:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205939456, offset 0 00000400:00000200:2.0:1788205205.737933:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205205.737937:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.737940:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205205.737943:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205205.737946:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205205.737949:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205205.737951:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205205.737954:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59195:59195:256:4294967295] 192.168.201.151@tcp LPNI seq info [59175:59175:8:4294967295] 00000400:00000200:2.0:1788205205.737958:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.737965:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205205.737981:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.737985:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205205.737989:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205205.737992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788205205.737998:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205205.738000:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788205205.738004:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.738006:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205205.738009:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205205.738013:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205205.738015:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205205.738017:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205205.738019:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205205.738023:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738030:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738033:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738040:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738042:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205205.738044:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205205.738048:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:2.0:1788205205.738052:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205205.738054:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301333147 00000100:00000001:2.0:1788205205.738057:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738059:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301333147 00000100:00000001:2.0:1788205205.738061:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738063:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205205.738065:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738067:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205205.738071:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205205/real 1788205205] req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205205.738078:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205205.738079:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205205.738081:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205205.738083:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205205.738084:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205205.738086:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205205.738089:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205205.738097:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738103:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205205.738104:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738106:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738107:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205205.738111:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738117:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205205.738120:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738128:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205205.738131:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738133:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738134:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738136:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205205.738138:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205205.738141:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205205.738144:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205205.738145:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205205.738149:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205205.738157:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939456:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205205.738161:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205205.738164:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738170:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205205.738172:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.738174:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205205.738176:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205205.738178:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205205.738180:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939584:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205205.738183:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205205.738185:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205205.738187:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.738188:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205205.738191:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205205.738193:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205205.738196:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205205.738199:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205205.738202:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205205.738205:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939584, portal 4 00000100:00000001:2.0:1788205205.738206:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205205.738208:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:2.0:1788205205.738214:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738220:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205205.738222:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205205.738224:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205205.738226:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205939584, offset 0 00000400:00000200:2.0:1788205205.738230:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205205.738233:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.738237:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205205.738240:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205205.738242:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205205.738244:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205205.738246:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205205.738249:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59196:59196:256:4294967295] 192.168.201.151@tcp LPNI seq info [59176:59176:8:4294967295] 00000400:00000200:2.0:1788205205.738253:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.738260:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205205.738264:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.738266:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205205.738270:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205205.738273:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788205205.738277:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205205.738280:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788205205.738283:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.738285:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205205.738288:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205205.738291:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205205.738293:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205205.738295:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205205.738297:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205205.738301:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738307:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738311:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738318:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738319:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205205.738321:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205205.738324:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:2.0:1788205205.738327:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205205.738329:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301333033 00000100:00000001:2.0:1788205205.738332:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738334:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301333033 00000100:00000001:2.0:1788205205.738336:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738338:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205205.738340:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738342:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205205.738345:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205205/real 1788205205] req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205205.738353:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205205.738354:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205205.738355:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205205.738357:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205205.738359:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205205.738361:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205205.738382:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205205.738392:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738398:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205205.738399:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738401:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738402:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205205.738406:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738412:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205205.738416:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738423:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205205.738425:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738427:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738428:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738430:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205205.738432:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205205.738434:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205205.738437:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205205.738439:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205205.738443:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205205.738450:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939584:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205205.738454:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205205.738457:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738464:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205205.738465:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.738468:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205205.738469:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205205.738471:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205205.738474:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205205.738477:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205205.738479:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205205.738481:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205205.738483:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205205.738486:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788205205.738488:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205205.738491:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:2.0:1788205205.738493:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3181aa688. 00000400:00000010:2.0:1788205205.738495:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d853b8. 00000100:00000200:2.0:1788205205.738498:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939712, portal 4 00000100:00000001:2.0:1788205205.738500:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205205.738501:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:2.0:1788205205.738506:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738512:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205205.738514:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:2.0:1788205205.738517:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c310d856e8. 00000100:00000200:2.0:1788205205.738519:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205939712, offset 0 00000400:00000200:2.0:1788205205.738523:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205205.738526:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.738529:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:2.0:1788205205.738532:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788205205.738534:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:2.0:1788205205.738537:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788205205.738539:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205205.738542:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59197:59197:256:4294967295] 192.168.201.151@tcp LPNI seq info [59177:59177:8:4294967295] 00000400:00000200:2.0:1788205205.738546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205205.738552:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:2.0:1788205205.738557:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.738559:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3069d4f00. 00000800:00000200:2.0:1788205205.738563:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.0:1788205205.738565:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1788205205.738569:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:2.1:1788205205.738572:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1788205205.738575:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:2.0:1788205205.738578:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3069d4f00. 00000400:00000200:2.0:1788205205.738580:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1788205205.738583:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205205.738586:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d856e8 00000400:00000010:2.0:1788205205.738587:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d856e8. 00000100:00000001:2.0:1788205205.738590:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205205.738593:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738600:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738603:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205230 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738610:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738612:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205205.738614:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205205.738617:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:2.0:1788205205.738621:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205205.738622:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301333040 00000100:00000001:2.0:1788205205.738624:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738626:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301333040 00000100:00000001:2.0:1788205205.738629:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301337090 00000100:00000001:2.0:1788205205.738631:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205205.738633:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738635:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205205.738638:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205205/real 1788205205] req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788205205.738645:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788205205.738646:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1788205205.738648:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3181aa688. 00000400:00000200:2.0:1788205205.738650:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c310d853b8 00000400:00000010:2.0:1788205205.738652:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c310d853b8. 00000100:00000001:2.0:1788205205.738654:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1788205205.738657:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205205.738665:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738672:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205205.738673:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738675:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738676:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205205.738679:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738684:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205205.738688:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738695:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205205.738697:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738699:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205205.738700:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738702:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205205.738704:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205205.738706:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205205.738710:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:2.0:1788205205.738712:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205205.738715:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205205.738724:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939712:192.168.201.151@tcp:8:kworker.0 00000100:00000001:2.0:1788205205.738727:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738729:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738732:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205939456/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738739:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205205.738740:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205205.738742:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0e800. 02000000:00000001:2.0:1788205205.738744:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205205.738746:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205205.738747:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:2.0:1788205205.738749:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205205.738751:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:2.0:1788205205.738753:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205205.738754:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205205.738756:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:2.0:1788205205.738758:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205205.738760:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738761:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738764:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205939584/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738771:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205205.738772:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205205.738773:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0d400. 02000000:00000001:2.0:1788205205.738775:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205205.738776:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205205.738777:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:2.0:1788205205.738779:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205205.738781:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:2.0:1788205205.738782:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205205.738784:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205205.738785:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:2.0:1788205205.738786:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205205.738787:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738789:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205205.738791:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205939712/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205230 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205205.738797:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205205.738799:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205205.738800:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c305a0f400. 02000000:00000001:2.0:1788205205.738803:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205205.738804:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1788205205.738805:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:2.0:1788205205.738807:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1788205205.738808:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:2.0:1788205205.738810:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205205.738811:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205205.738812:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:2.0:1788205205.738814:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205205.738815:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738817:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.738820:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205205.738821:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205205.738822:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205205.738824:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205205.928493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205205.928499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205205.928505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205205.928507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205205.928510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205205.928511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205205.928515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205205.928516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205205.928653:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205205.928657:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205205.928659:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205206.249159:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205206.249167:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205206.249170:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205206.249176:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205206.249179:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205206.249183:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.249186:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205206.249189:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.249192:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205206.696473:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205206.696480:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205206.696482:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205206.696488:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205206.696491:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205206.696495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.696497:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205206.696499:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.696502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205206.760454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205206.760461:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205206.760464:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205206.760469:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205206.760472:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205206.760476:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.760479:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205206.760481:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.760484:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205206.760496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205206.760499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205206.760500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205206.760503:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205206.760505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205206.760507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205206.760509:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205206.760510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205206.760512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205206.952470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205206.952476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205206.952479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205206.952480:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205206.952483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205206.952484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205206.952486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205206.952486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205206.952537:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205206.952540:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205206.952543:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205207.272429:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205207.272436:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205207.272438:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205207.272444:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205207.272447:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205207.272451:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.272453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205207.272456:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.272459:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205207.720489:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205207.720496:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205207.720498:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205207.720504:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205207.720507:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205207.720511:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.720513:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205207.720516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.720518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205207.784442:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205207.784450:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205207.784453:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205207.784459:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205207.784463:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205207.784468:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.784471:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205207.784474:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.784477:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205207.784485:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205207.784490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205207.784492:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205207.784496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205207.784499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205207.784502:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205207.784504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205207.784507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205207.784509:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205207.976476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205207.976483:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205207.976488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205207.976490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205207.976493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205207.976494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205207.976497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205207.976498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205207.977329:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205207.977333:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205207.977337:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205208.296488:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205208.296494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205208.296495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205208.296499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205208.296501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205208.296504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.296506:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205208.296508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.296510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.744470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205208.744481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205208.744483:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205208.744487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205208.744490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205208.744493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.744494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205208.744496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.744498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.808424:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205208.808430:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205208.808431:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205208.808436:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205208.808439:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205208.808441:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.808443:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205208.808445:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.808447:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.808471:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205208.808473:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205208.808474:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205208.808475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205208.808477:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205208.808478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.808480:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205208.808482:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205208.808483:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205209.000512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205209.000518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205209.000522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205209.000524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205209.000526:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205209.000528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205209.000531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205209.000533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205209.000585:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205209.000588:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205209.000591:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205209.320487:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205209.320494:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205209.320496:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205209.320502:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205209.320505:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205209.320509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.320512:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205209.320514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.320517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205209.768500:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205209.768507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205209.768509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205209.768514:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:3.0:1788205209.768518:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205209.768521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.768524:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205209.768526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.768529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205209.833434:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205209.833442:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205209.833444:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205209.833449:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205209.833452:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205209.833604:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.833607:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205209.833617:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.833620:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205209.835623:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205209.835629:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205209.835631:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205209.835635:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205209.835639:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205209.835641:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205209.835644:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205209.835646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205209.835649:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205210.024515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205210.024521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205210.024527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205210.024529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205210.024531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205210.024533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205210.024536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205210.024537:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205210.024641:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205210.024645:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205210.024647:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205210.344506:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301337090, at 4301337600 00000100:00000010:0.0:1788205210.344530:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c17e0. 00000100:00000040:1.0:1788205210.344588:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301337090, at 4301337600 00000100:00000010:1.0:1788205210.344602:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3103c1cc0. 00000100:00000001:3.0:1788205210.344968:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205210.344972:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205210.344975:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205210.344978:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205210.344981:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205210.344985:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.344987:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205210.344990:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.344993:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205210.793631:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205210.793637:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205210.793640:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205210.793645:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205210.793649:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205210.793653:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205210.793655:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205210.793658:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205210.793661:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205210.856586:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205210.856597:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205210.856600:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205210.856603:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205210.856607:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205210.856615:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6665 00000400:00000200:2.0:1788205210.856623:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205210.856630:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205210.856633:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205210.856636:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856639:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.856642:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856644:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205210.856649:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205210.856651:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205210.856654:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205210.856657:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205210.856660:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205210.856662:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205210.856665:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856667:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.856668:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856671:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205210.856673:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205210.856676:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856679:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205210.856682:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205210.856685:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205210.856687:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.856690:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205210.856692:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205210.856701:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205210.856704:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205210.856709:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205210.856711:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.856714:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205210.856717:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205210.856721:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205210.856728:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788205210.856734:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205210.856743:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205210.856753:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205210.856757:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205210.856761:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205210.856784:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205210.856787:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205210.856791:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205210.856792:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205210.856794:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205210.856796:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205210.856799:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6665 00000400:00000200:2.0:1788205210.856802:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205210.856805:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205210.856808:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205210.856813:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856815:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.856817:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856819:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205210.856822:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205210.856824:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205210.856826:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205210.856828:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205210.856831:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205210.856833:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205210.856836:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856838:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.856840:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856842:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205210.856844:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:0.0:1788205210.856848:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788205210.856848:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205210.856856:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.856860:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788205210.856860:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205210.856867:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788205210.856870:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205210.856875:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.856877:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:0.0:1788205210.856900:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.856915:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788205210.856917:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788205210.856919:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205210.856931:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.856932:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205210.856936:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.856939:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1788205210.856943:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 02000000:00000001:0.0:1788205210.856944:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205210.856953:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00100000:0.0:1788205210.856957:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205939840:192.168.201.151@tcp:250:kworker.0 02000000:00000001:2.0:1788205210.856960:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205210.856966:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.856969:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205210.856972:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205210.856975:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205210.856977:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205210.856978:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.856983:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.856989:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1788205210.856992:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 02000000:00000010:0.0:1788205210.857000:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 00000100:00000001:2.0:1788205210.857000:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.857006:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205210.857013:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205210.857026:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1788205210.857028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205210.857033:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205210.857040:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000100:00000040:2.0:1788205210.857041:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788205210.857053:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000001:2.0:1788205210.857065:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205210.857067:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939840, portal 25 00000100:00080000:2.0:1788205210.857070:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205210.857072:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788205210.857075:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205210.857077:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788205210.857080:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205210.857085:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1788205210.857092:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205210.857093:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205210.857104:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6665 00000400:00000200:2.0:1788205210.857114:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205210.857119:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1788205210.857121:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205210.857126:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205210.857132:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000040:2.0:1788205210.857138:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:0.0:1788205210.857140:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000001:3.0:1788205210.857140:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205210.857143:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205210.857147:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1788205210.857151:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205939840, offset 0 00000100:00000001:1.0:1788205210.857151:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205210.857151:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.857152:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205210.857158:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205210.857163:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205210.857165:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205210.857168:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205210.857170:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205210.857172:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205210.857174:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.857176:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205210.857180:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.857188:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.857189:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205210.857189:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205210.857197:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205210.857201:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.857218:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.857229:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205210.857236:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205210.857241:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205210.857246:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205210.857250:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205210.857254:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59198:59198:256:4294967295] 192.168.201.151@tcp LPNI seq info [59178:59178:8:4294967295] 00000400:00000200:0.0:1788205210.857259:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.857275:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205210.857283:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.857290:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205210.857303:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205210.857308:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788205210.857315:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205210.857319:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788205210.857324:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.857327:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205210.857338:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205210.857344:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205210.857350:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205210.857353:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205210.857359:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205210.857396:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857413:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.857417:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857426:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.857428:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205210.857437:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.857441:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205210.857445:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301342209 00000100:00000010:0.0:1788205210.857451:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:0.0:1788205210.857459:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205210.857466:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301342209 00000100:00000010:0.0:1788205210.857470:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff98a0. 00000100:00000001:0.0:1788205210.857473:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205210.857484:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205210.857488:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.857491:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205210.857496:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205210/real 1788205210] req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205210.857515:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205210.857518:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205210.857520:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205210.857523:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205210.857524:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205210.857528:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205210.857531:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205210.857540:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857548:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205210.857550:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857552:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.857555:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205210.857559:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857567:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205210.857572:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857580:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205210.857583:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857585:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.857586:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857589:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205210.857592:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205210.857603:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205210.857607:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205210.857610:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205210.857615:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205210.857624:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205939840:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205210.857629:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857632:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.857634:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205939840/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857639:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205210.857642:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205210.857644:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205210.857647:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205210.857649:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205210.857650:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205210.857652:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205210.857654:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:0.0:1788205210.857664:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205210.857666:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205210.857668:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205210.857672:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205210.857673:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857676:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857684:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205210.857686:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.857688:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205210.857690:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205210.857692:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205210.857697:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857711:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205210.857713:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.857715:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205210.857716:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205210.857719:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205210.857722:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939968:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205210.857725:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205210.857728:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205210.857729:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.857731:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205210.857735:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205210.857737:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.857741:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205210.857744:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205210.857748:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205210.857751:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205939968, portal 10 00000100:00000001:0.0:1788205210.857753:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205210.857754:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205210.857760:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857764:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205210.857766:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205210.857767:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205210.857769:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205939968, offset 0 00000400:00000200:0.0:1788205210.857771:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205210.857774:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.857777:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205210.857779:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205210.857781:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205210.857783:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205210.857784:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205210.857786:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59199:59199:256:4294967295] 192.168.201.151@tcp LPNI seq info [59179:59179:8:4294967295] 00000400:00000200:0.0:1788205210.857789:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.857793:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205210.857796:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.857798:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205210.857801:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205210.857803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788205210.857806:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205210.857807:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788205210.857810:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.857811:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205210.857813:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205210.857816:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205210.857818:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205210.857820:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205210.857822:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205210.857825:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857833:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.857835:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857842:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.857843:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205210.857846:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205210.857850:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205210.857854:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205210.857856:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301338266 00000100:00000001:0.0:1788205210.857859:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.857862:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301338266 00000100:00000001:0.0:1788205210.857864:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.857866:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205210.857868:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.857870:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205210.857874:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205210/real 1788205210] req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205210.857892:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205210.857894:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205210.857896:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205210.857899:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205210.857900:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205210.857903:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205210.857906:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205210.857915:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857921:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205210.857923:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857925:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.857926:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205210.857930:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857936:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205210.857940:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857947:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205210.857950:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857952:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.857953:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857955:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205210.857958:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205210.857960:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205210.857964:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205210.857966:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205210.857969:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205210.857974:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205939968:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205210.857977:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.857978:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.857980:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205939968/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.857983:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205210.857984:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205210.857986:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205210.857988:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205210.857989:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205210.857990:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205210.857991:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205210.857992:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:0.0:1788205210.857994:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205210.857995:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205210.857996:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205210.857998:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205210.857999:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.858000:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.858003:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205210.858004:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.858005:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205210.858006:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205210.858705:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858709:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205210.858713:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205210.858715:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205210.858717:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205210.858720:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205210.858722:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205210.858725:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205210.858729:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858731:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.858734:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858736:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205210.858738:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205210.858741:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858744:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205210.858747:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205210.858753:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205210.858755:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858757:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205210.858761:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205210.858764:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858766:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205210.858774:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205210.858777:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205210.858780:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205210.858782:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858785:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205210.858786:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205210.858789:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205210.858796:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205210.858801:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205210.858808:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205210.858817:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205210.858819:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205210.858823:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205210.858850:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205210.858853:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205210.858857:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205210.858859:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205210.858860:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205210.858863:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205210.858866:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6665 00000400:00000200:2.0:1788205210.858873:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205210.858877:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205210.858879:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205210.858893:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858895:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.858897:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858899:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205210.858901:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205210.858902:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205210.858903:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205210.858905:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205210.858907:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205210.858909:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205210.858911:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858913:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205210.858915:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858917:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205210.858919:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205210.858921:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205210.858924:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205210.858925:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205210.858929:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205210.858930:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858932:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205210.858934:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205210.858936:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858937:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205210.858940:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205210.858942:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205210.858945:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205210.858947:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.858948:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205210.858949:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205210.858951:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205210.858954:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788205210.858957:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205210.858961:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205210.858967:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205210.858969:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205210.858972:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205210.858978:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205210.858983:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6675) 00000100:00000001:0.0:1788205210.859129:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205210.859133:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205210.859136:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205210.859142:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859149:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205210.859150:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.859154:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205210.859168:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205210.859171:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205210.859174:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940096:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205210.859179:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205210.859183:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205210.859185:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.859187:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205210.859192:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205210.859194:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.859200:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205210.859203:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205210.859209:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205210.859215:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940096, portal 4 00000100:00000001:0.0:1788205210.859218:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205210.859219:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205210.859225:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859232:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205210.859234:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205210.859236:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205210.859239:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205940096, offset 0 00000400:00000200:0.0:1788205210.859244:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205210.859253:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.859381:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205210.859387:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205210.859391:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205210.859395:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205210.859398:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205210.859401:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59200:59200:256:4294967295] 192.168.201.151@tcp LPNI seq info [59180:59180:8:4294967295] 00000400:00000200:0.0:1788205210.859406:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.859415:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205210.859421:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.859427:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205210.859433:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205210.859437:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788205210.859443:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205210.859446:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788205210.859450:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.859453:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205210.859460:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205210.859465:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205210.859542:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205210.859544:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205210.859549:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205210.859553:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859562:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.859565:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859571:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.859573:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205210.859578:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205210.859582:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205210.859585:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205210.859587:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301338155 00000100:00000001:0.0:1788205210.859590:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.859593:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301338155 00000100:00000001:0.0:1788205210.859595:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.859598:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205210.859600:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.859602:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205210.859607:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205210/real 1788205210] req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205210.859615:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205210.859617:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205210.859619:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205210.859621:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205210.859623:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205210.859626:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205210.859629:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205210.859636:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859642:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205210.859643:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859645:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.859647:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205210.859651:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859657:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205210.859661:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859666:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205210.859668:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859669:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.859670:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859673:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205210.859675:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205210.859678:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205210.859680:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205210.859682:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205210.859686:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205210.859692:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940096:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205210.859695:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205210.859699:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859704:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205210.859705:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.859707:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205210.859709:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205210.859710:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205210.859713:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205210.859716:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205210.859718:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205210.859719:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205210.859721:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205210.859724:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205210.859726:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205210.859729:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205210.859731:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205210.859734:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205210.859736:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940224, portal 4 00000100:00000001:0.0:1788205210.859738:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205210.859739:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:0.0:1788205210.859744:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859749:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205210.859751:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205210.859752:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205210.859754:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205940224, offset 0 00000400:00000200:0.0:1788205210.859758:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205210.859762:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.859765:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205210.859768:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205210.859770:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205210.859772:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205210.859773:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205210.859776:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59201:59201:256:4294967295] 192.168.201.151@tcp LPNI seq info [59181:59181:8:4294967295] 00000400:00000200:0.0:1788205210.859779:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205210.859784:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205210.859788:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.859791:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205210.859794:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205210.859797:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788205210.859800:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205210.859803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788205210.859806:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205210.859808:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205210.859811:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205210.859813:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205210.859816:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205210.859817:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205210.859820:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205210.859823:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859830:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.859832:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205235 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859842:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.859843:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205210.859846:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205210.859848:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205210.859852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205210.859854:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301338161 00000100:00000001:0.0:1788205210.859856:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.859858:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301338161 00000100:00000001:0.0:1788205210.859859:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301342209 00000100:00000001:0.0:1788205210.859861:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205210.859863:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.859864:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205210.859867:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205210/real 1788205210] req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205210.859874:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205210.859875:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205210.859876:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205210.859878:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205210.859880:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205210.859896:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205210.859899:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205210.859906:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859911:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205210.859913:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859914:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.859915:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205210.859918:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859924:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205210.859927:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859933:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205210.859935:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859936:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205210.859938:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859939:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205210.859941:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205210.859943:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205210.859946:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205210.859948:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205210.859951:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205210.859957:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940224:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205210.859961:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859963:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.859965:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205940096/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.859971:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205210.859973:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205210.859975:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205210.859977:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205210.859980:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205210.859981:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205210.859983:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205210.859985:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205210.859990:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205210.859991:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205210.859993:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205210.859996:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205210.859997:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.859999:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205210.860002:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205940224/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205235 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205210.860008:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205210.860009:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205210.860011:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205210.860012:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205210.860013:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205210.860014:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205210.860015:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205210.860017:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205210.860018:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205210.860019:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205210.860020:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205210.860022:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205210.860023:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.860025:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205210.860032:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205210.860034:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205210.860036:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205210.860038:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205211.048536:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205211.048545:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205211.048548:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205211.048552:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048555:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205211.048558:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048559:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048562:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048565:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205211.048567:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205211.048570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205211.048572:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205211.048633:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205211.048637:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205211.048641:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205211.368481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205211.368488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205211.368490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205211.368495:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205211.368498:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205211.368502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.368504:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205211.368507:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.368510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205211.816743:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205211.816751:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205211.816753:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205211.816758:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205211.816761:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205211.816765:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205211.816767:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205211.816770:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205211.816772:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205211.880475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205211.880483:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205211.880485:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205211.880490:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205211.880494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205211.880498:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.880501:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205211.880503:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.880506:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205211.880757:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205211.880760:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205211.880761:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205211.880765:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205211.880767:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205211.880769:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205211.880771:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205211.880773:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205211.880775:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205212.073136:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205212.073141:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205212.073145:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205212.073146:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205212.073147:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205212.073148:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205212.073150:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205212.073151:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205212.073451:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205212.073455:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205212.073458:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205212.392485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205212.392492:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205212.392494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205212.392499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205212.392502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205212.392505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205212.392508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205212.392510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205212.392513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.840490:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205212.840499:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205212.840501:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.840506:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205212.840510:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205212.840514:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.840516:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.840518:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.840521:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205212.904587:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205212.904593:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205212.904594:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205212.904598:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205212.904600:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205212.904602:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205212.904604:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205212.904606:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205212.904608:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.904657:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205212.904659:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205212.904660:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.904662:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205212.904664:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205212.904665:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.904666:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.904667:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.904668:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098139:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205213.098151:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205213.098153:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205213.098158:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098164:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205213.098166:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098168:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098179:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098185:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205213.098186:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098189:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205213.098191:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205213.098192:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205213.098194:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098195:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205213.098196:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098198:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098199:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098202:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205213.098204:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205213.098205:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205213.098206:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098208:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205213.098209:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098210:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205213.098212:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205213.098297:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205213.098302:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205213.098307:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205213.416489:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205213.416496:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205213.416498:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205213.416503:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205213.416507:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205213.416511:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.416514:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205213.416517:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.416519:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205213.864469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205213.864475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205213.864477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205213.864481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205213.864484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205213.864486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205213.864489:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205213.864491:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205213.864493:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205213.928474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205213.928480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205213.928482:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205213.928486:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205213.928488:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205213.928491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205213.928493:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205213.928495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205213.928497:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205213.928569:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205213.928572:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205213.928573:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205213.928576:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205213.928578:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205213.928579:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.928581:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205213.928582:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.928584:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205214.120448:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205214.120454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205214.120459:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205214.120460:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205214.120462:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205214.120463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205214.120466:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205214.120467:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205214.120521:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205214.120525:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205214.120528:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205214.440478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205214.440483:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205214.440485:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205214.440489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205214.440493:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205214.440497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.440500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205214.440502:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.440505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205214.888418:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205214.888424:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205214.888425:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205214.888429:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205214.888431:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205214.888434:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205214.888436:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205214.888438:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205214.888441:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205214.952469:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205214.952469:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205214.952476:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205214.952477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205214.952479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205214.952479:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205214.952483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205214.952483:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205214.952487:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205214.952487:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205214.952490:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205214.952491:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.952493:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205214.952496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205214.952498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205214.952498:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205214.952500:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205214.952502:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205215.144474:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205215.144479:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205215.144483:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205215.144484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205215.144486:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205215.144487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205215.144488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205215.144489:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205215.144533:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205215.144537:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205215.144539:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205215.464423:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205215.464434:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205215.464436:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205215.464440:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000040:1.0:1788205215.464441:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301342209, at 4301342720 00000100:00000040:0.0:1788205215.464442:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301342209, at 4301342720 00000100:00000001:3.0:1788205215.464443:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205215.464448:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205215.464451:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1788205215.464452:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff98a0. 00000100:00000010:0.0:1788205215.464454:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000001:3.0:1788205215.464454:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205215.464459:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205215.912476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205215.912482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205215.912484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205215.912487:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205215.912490:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205215.912493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205215.912494:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205215.912496:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205215.912498:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205215.976462:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1788205215.976462:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205215.976462:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205215.976468:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205215.976469:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205215.976471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205215.976473:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205215.976475:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788205215.976475:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1788205215.976478:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205215.976480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205215.976482:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205215.976484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205215.976486:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205215.976486:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205215.976488:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205215.976492:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205215.976493:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205215.976495:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205215.976495:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205215.976500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205215.976501:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205215.976504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205215.976511:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6670 00000400:00000200:2.0:1788205215.976520:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205215.976526:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205215.976530:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205215.976533:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976537:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205215.976539:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976542:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205215.976545:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205215.976547:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205215.976550:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205215.976553:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205215.976556:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205215.976558:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205215.976561:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976562:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205215.976564:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976566:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205215.976570:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205215.976573:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976575:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205215.976578:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205215.976581:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205215.976585:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976587:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205215.976589:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205215.976597:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205215.976601:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205215.976605:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205215.976607:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976609:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205215.976611:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205215.976616:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205215.976623:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:2.0:1788205215.976629:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205215.976638:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205215.976648:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205215.976652:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205215.976656:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.976677:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205215.976680:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205215.976684:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205215.976685:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205215.976686:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205215.976689:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205215.976692:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6670 00000400:00000200:2.0:1788205215.976696:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205215.976700:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205215.976703:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205215.976706:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788205215.976707:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205215.976709:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205215.976711:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205215.976712:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205215.976716:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976719:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:0.0:1788205215.976722:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205215.976723:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205215.976726:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205215.976729:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205215.976732:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205215.976736:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205215.976738:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788205215.976740:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205215.976742:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205215.976747:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205215.976748:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205215.976752:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205215.976753:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788205215.976757:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 02000000:00000001:0.0:1788205215.976758:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976762:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:0.0:1788205215.976766:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205940352:192.168.201.151@tcp:250:kworker.0 00000020:00000040:2.0:1788205215.976769:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:0.0:1788205215.976771:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788205215.976775:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:0.0:1788205215.976776:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205215.976779:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976780:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.976781:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205215.976785:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1788205215.976788:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205215.976791:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205215.976791:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205215.976813:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976815:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205215.976819:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1788205215.976820:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205215.976826:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000008:00000001:2.0:1788205215.976826:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.976831:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205215.976835:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000010:2.0:1788205215.976837:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000100:00000200:0.0:1788205215.976841:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940352, portal 25 00000020:00000040:2.0:1788205215.976842:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:0.0:1788205215.976845:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205215.976847:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 02000000:00000001:2.0:1788205215.976848:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205215.976851:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.976854:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.976855:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205215.976858:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205215.976861:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205215.976868:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:0.0:1788205215.976869:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205215.976874:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.976875:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205215.976880:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00100000:2.0:1788205215.976883:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1788205215.976885:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205940352, offset 0 00000100:00000001:2.0:1788205215.976896:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1788205215.976897:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976900:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205215.976907:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205215.976908:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205215.976915:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205215.976921:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788205215.976922:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.976927:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:2.0:1788205215.976927:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1788205215.976931:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788205215.976934:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:0.0:1788205215.976935:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976939:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1788205215.976940:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59202:59202:256:4294967295] 192.168.201.151@tcp LPNI seq info [59182:59182:8:4294967295] 00000100:00080000:2.0:1788205215.976943:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1788205215.976945:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205215.976950:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1788205215.976958:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788205215.976958:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6670 00000800:00000200:0.0:1788205215.976967:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000400:00000200:2.0:1788205215.976967:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205215.976973:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205215.976976:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000010:0.0:1788205215.976979:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000100:00000040:2.0:1788205215.976980:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.976984:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1788205215.976988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000040:2.0:1788205215.976990:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:0.0:1788205215.976992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000100:00000001:2.0:1788205215.976996:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:0.1:1788205215.977000:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788205215.977002:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:0.1:1788205215.977004:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000020:00000040:2.0:1788205215.977005:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205215.977009:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1788205215.977010:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000040:2.0:1788205215.977013:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000800:00000010:0.0:1788205215.977014:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000020:00000001:2.0:1788205215.977019:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205215.977022:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1788205215.977022:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205215.977030:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205215.977033:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205215.977037:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788205215.977038:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205215.977043:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000100:00000001:2.0:1788205215.977043:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000010:0.0:1788205215.977046:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000020:00000040:2.0:1788205215.977048:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00000001:0.0:1788205215.977051:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788205215.977055:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000200:0.0:1788205215.977057:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.977059:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205215.977064:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205215.977069:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205215.977070:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788205215.977074:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.977075:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.977081:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205215.977085:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205215.977086:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:0.0:1788205215.977087:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1788205215.977091:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977093:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.977094:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.977098:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:2.0:1788205215.977100:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205215.977101:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301347329 00000020:00000040:2.0:1788205215.977105:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000010:0.0:1788205215.977107:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 02000000:00000001:2.0:1788205215.977110:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205215.977113:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977114:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205215.977118:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788205215.977121:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301347329 02000000:00000001:2.0:1788205215.977122:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:0.0:1788205215.977127:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9600. 02000000:00000001:2.0:1788205215.977127:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977132:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 02000000:00000010:2.0:1788205215.977134:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:0.0:1788205215.977135:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205215.977138:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.977141:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977142:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205215.977150:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788205215.977153:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205215/real 1788205215] req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205215.977162:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205215.977163:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205215.977165:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205215.977166:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205215.977167:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205215.977169:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205215.977173:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205215.977198:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.977201:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205215.977206:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205215.977207:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977209:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977212:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.977213:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205215.977217:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205215.977219:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.977232:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977238:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205215.977238:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1788205215.977244:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205215.977246:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205215.977250:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205215.977253:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205215.977254:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205215.977257:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977259:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205215.977260:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205215.977261:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977264:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205215.977267:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205215.977267:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6670 00000100:00080000:0.0:1788205215.977274:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1788205215.977277:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788205215.977279:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205215.977282:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:2.0:1788205215.977285:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000040:0.0:1788205215.977287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205215.977289:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205215.977298:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00100000:0.0:1788205215.977300:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205940352:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205215.977302:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205215.977305:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977308:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205215.977308:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:0.0:1788205215.977313:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205940352/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205215.977314:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205215.977319:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205215.977321:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205215.977325:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788205215.977326:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205215.977329:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1788205215.977329:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 02000000:00000010:0.0:1788205215.977332:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 00000020:00000001:2.0:1788205215.977334:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 02000000:00000001:0.0:1788205215.977335:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205215.977339:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1788205215.977339:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000040:0.0:1788205215.977341:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205215.977344:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:2.0:1788205215.977346:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000010:0.0:1788205215.977347:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:0.0:1788205215.977350:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205215.977351:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205215.977352:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205215.977353:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205215.977354:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977355:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977361:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205215.977362:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205215.977388:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977392:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205215.977395:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205215.977396:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205215.977400:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205215.977400:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:0.0:1788205215.977406:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205215.977406:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205215.977414:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205215.977422:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205215.977426:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205215.977427:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205215.977429:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977431:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205215.977431:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205215.977434:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205215.977437:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205215.977437:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205215.977443:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940480:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205215.977443:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205215.977448:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788205215.977448:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 02000000:00000001:0.0:1788205215.977450:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205215.977452:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977454:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788205215.977454:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205215.977457:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 00000100:00000001:2.0:1788205215.977458:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977460:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.977464:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000010:2.0:1788205215.977465:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000400:00000010:0.0:1788205215.977467:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000020:00000040:2.0:1788205215.977470:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000010:0.0:1788205215.977472:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 02000000:00000001:2.0:1788205215.977476:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1788205215.977477:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940480, portal 10 00000100:00000001:0.0:1788205215.977480:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788205215.977480:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977482:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000001:2.0:1788205215.977485:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205215.977488:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1788205215.977489:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205215.977492:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977498:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205215.977500:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000010:2.0:1788205215.977501:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000400:00000010:0.0:1788205215.977502:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205215.977506:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205940480, offset 0 00000100:00000001:2.0:1788205215.977507:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.977510:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205215.977514:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:2.0:1788205215.977518:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1788205215.977519:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205215.977524:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205215.977527:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205215.977531:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205215.977535:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205215.977540:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59203:59203:256:4294967295] 192.168.201.151@tcp LPNI seq info [59183:59183:8:4294967295] 00000100:00000001:2.0:1788205215.977541:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1788205215.977546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205215.977546:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205215.977555:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205215.977556:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205215.977565:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.977570:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000100:00000001:2.0:1788205215.977577:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205215.977582:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6680) 00000800:00000200:0.0:1788205215.977583:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205215.977588:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788205215.977592:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205215.977594:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788205215.977597:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.977598:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205215.977601:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205215.977604:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205215.977607:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205215.977608:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205215.977610:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205215.977612:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977617:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.977619:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977623:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977624:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205215.977625:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.977628:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205215.977630:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205215.977631:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301343387 00000100:00000001:0.0:1788205215.977633:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.977634:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301343387 00000100:00000001:0.0:1788205215.977635:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.977636:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205215.977638:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977639:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205215.977641:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205215/real 1788205215] req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205215.977646:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205215.977646:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205215.977647:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205215.977648:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205215.977649:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205215.977651:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205215.977652:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205215.977657:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977660:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205215.977661:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977662:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.977663:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205215.977665:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977668:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205215.977670:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977674:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205215.977675:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977676:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.977677:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977677:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205215.977679:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205215.977680:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205215.977682:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205215.977683:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205215.977686:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205215.977690:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940480:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205215.977692:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205215.977699:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977704:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205215.977705:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977706:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205215.977707:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205215.977708:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205215.977710:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940608:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205215.977712:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205215.977713:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205215.977714:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977715:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205215.977718:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205215.977719:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.977722:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205215.977724:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205215.977727:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205215.977730:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940608, portal 4 00000100:00000001:0.0:1788205215.977732:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205215.977733:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205215.977738:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977744:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205215.977746:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205215.977749:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205215.977751:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205940608, offset 0 00000400:00000200:0.0:1788205215.977754:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205215.977758:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205215.977761:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205215.977764:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205215.977766:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205215.977768:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205215.977769:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205215.977771:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59204:59204:256:4294967295] 192.168.201.151@tcp LPNI seq info [59184:59184:8:4294967295] 00000400:00000200:0.0:1788205215.977774:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205215.977779:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205215.977783:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.977786:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205215.977790:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205215.977803:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788205215.977808:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205215.977811:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788205215.977814:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.977816:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205215.977819:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205215.977823:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205215.977826:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205215.977828:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205215.977830:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205215.977834:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977841:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.977844:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977849:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977851:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205215.977855:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.977858:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205215.977861:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205215.977862:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301343273 00000100:00000001:0.0:1788205215.977865:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.977866:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301343273 00000100:00000001:0.0:1788205215.977868:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.977870:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205215.977872:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977874:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205215.977878:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205215/real 1788205215] req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205215.977885:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205215.977886:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205215.977888:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205215.977889:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205215.977890:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205215.977892:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205215.977894:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205215.977898:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977902:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205215.977903:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977904:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.977905:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205215.977907:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977910:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205215.977912:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977917:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205215.977918:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977919:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.977920:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977921:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205215.977922:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205215.977924:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205215.977925:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205215.977927:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205215.977929:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205215.977933:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940608:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205215.977935:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977936:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.977938:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205940480/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977942:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205215.977942:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205215.977944:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205215.977945:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205215.977946:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205215.977947:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205215.977948:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205215.977949:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:0.0:1788205215.977951:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205215.977952:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205215.977952:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205215.977954:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205215.977955:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977956:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.977958:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205940608/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977961:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205215.977962:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205215.977963:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205215.977964:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205215.977965:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205215.977965:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205215.977966:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205215.977967:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205215.977969:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205215.977969:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205215.977970:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205215.977971:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205215.977971:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977972:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.977975:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205215.977975:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.977977:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205215.977977:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205215.977978:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205215.977980:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.977983:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205215.977984:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977985:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205215.977986:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205215.977987:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205215.977989:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940736:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205215.977991:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205215.977992:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205215.977993:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205215.977994:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205215.977996:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205215.977998:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205215.978000:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205215.978001:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205215.978003:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205215.978005:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940736, portal 4 00000100:00000001:0.0:1788205215.978006:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205215.978007:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205215.978010:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978014:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205215.978015:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205215.978016:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205215.978018:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205940736, offset 0 00000400:00000200:0.0:1788205215.978020:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205215.978022:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205215.978024:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205215.978026:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205215.978027:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205215.978029:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205215.978030:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205215.978032:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59205:59205:256:4294967295] 192.168.201.151@tcp LPNI seq info [59185:59185:8:4294967295] 00000400:00000200:0.0:1788205215.978035:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205215.978038:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205215.978041:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.978042:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205215.978045:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205215.978046:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788205215.978049:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205215.978051:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788205215.978053:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205215.978054:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205215.978056:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205215.978058:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205215.978060:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205215.978061:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205215.978062:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205215.978064:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978068:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.978070:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205240 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978073:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.978074:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205215.978076:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205215.978077:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205215.978079:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205215.978080:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301343279 00000100:00000001:0.0:1788205215.978081:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.978083:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301343279 00000100:00000001:0.0:1788205215.978084:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301347329 00000100:00000001:0.0:1788205215.978085:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205215.978086:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.978087:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205215.978089:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205215/real 1788205215] req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205215.978093:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205215.978094:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205215.978094:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205215.978096:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205215.978097:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205215.978098:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205215.978099:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205215.978108:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978111:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205215.978112:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978113:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.978114:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205215.978115:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978119:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205215.978121:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978124:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205215.978125:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978126:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205215.978127:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978128:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205215.978129:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205215.978130:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205215.978132:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205215.978133:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205215.978135:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205215.978139:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940736:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205215.978141:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978142:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205215.978144:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205940736/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205240 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205215.978151:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205215.978152:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205215.978153:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205215.978154:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205215.978155:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205215.978155:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205215.978156:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205215.978157:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:0.0:1788205215.978158:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205215.978159:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205215.978160:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205215.978161:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205215.978162:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978163:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205215.978164:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205215.978165:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205215.978166:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205215.978167:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205216.168515:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205216.168522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205216.168528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205216.168529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205216.168532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205216.168533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205216.168537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205216.168538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205216.168602:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205216.168605:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205216.168607:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205216.488435:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205216.488441:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205216.488442:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205216.488447:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205216.488449:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205216.488453:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205216.488455:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205216.488456:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205216.488458:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205216.937956:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205216.937963:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205216.937966:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205216.937971:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205216.937975:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205216.937979:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205216.937981:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205216.937983:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205216.937986:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205217.000487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205217.000488:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205217.000496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205217.000496:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205217.000498:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.000500:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205217.000505:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205217.000506:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205217.000511:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205217.000512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205217.000516:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.000517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205217.000520:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205217.000525:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205217.000528:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.000529:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.000533:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.000536:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205217.192998:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205217.193004:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205217.193011:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205217.193012:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205217.193016:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205217.193017:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205217.193020:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205217.193022:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205217.193055:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205217.193059:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205217.193062:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205217.512878:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205217.512885:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205217.512888:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205217.512893:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205217.512896:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205217.512899:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205217.512902:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205217.512905:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205217.512908:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.960439:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205217.960446:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205217.960448:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205217.960454:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205217.960457:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205217.960461:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.960463:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.960465:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.960468:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205218.024852:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205218.024860:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205218.024862:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205218.024867:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205218.024871:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205218.024874:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205218.024877:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205218.024879:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205218.024882:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205218.026844:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205218.026850:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205218.026851:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205218.026856:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205218.026860:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205218.026862:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205218.026865:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205218.026867:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205218.026869:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205218.216880:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205218.216886:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205218.216891:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205218.216892:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205218.216896:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205218.216897:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205218.216901:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205218.216902:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205218.217016:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205218.217020:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205218.217023:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205218.536493:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205218.536500:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205218.536502:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205218.536507:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205218.536511:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205218.536515:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205218.536518:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205218.536521:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205218.536523:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205218.984542:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205218.984552:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205218.984554:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205218.984562:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205218.984566:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205218.984571:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205218.984574:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205218.984577:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205218.984581:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205219.048510:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205219.048519:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205219.048522:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205219.048528:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205219.048532:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205219.048537:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205219.048541:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205219.048544:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205219.048548:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205219.048588:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205219.048593:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205219.048594:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205219.048598:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205219.048600:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205219.048602:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205219.048605:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205219.048606:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205219.048609:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205219.240582:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205219.240592:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205219.240601:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205219.240603:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205219.240607:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205219.240609:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205219.240614:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205219.240615:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205219.240687:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205219.240693:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205219.240697:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205219.561940:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205219.561950:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205219.561952:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205219.561959:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205219.561963:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205219.561980:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205219.561984:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205219.561987:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205219.561990:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205220.008858:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205220.008868:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205220.008870:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205220.008876:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205220.008880:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205220.008885:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205220.008888:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205220.008891:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205220.008894:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205220.072497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205220.072497:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205220.072507:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205220.072508:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205220.072512:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205220.072513:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205220.072521:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205220.072521:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205220.072528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205220.072533:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205220.072535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205220.072538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205220.072542:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205220.074780:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205220.074790:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205220.074794:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205220.074797:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205220.074801:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205220.264502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205220.264508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205220.264514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205220.264515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205220.264518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205220.264519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205220.264521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205220.264522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205220.264583:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205220.264587:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205220.264590:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205220.585232:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301347329, at 4301347840 00000100:00000040:0.0:1788205220.585244:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301347329, at 4301347840 00000100:00000010:1.0:1788205220.585254:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9600. 00000100:00000010:0.0:1788205220.585260:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000001:3.0:1788205220.585276:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205220.585282:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205220.585284:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205220.585290:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205220.585295:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205220.585299:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205220.585303:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205220.585307:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205220.585310:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205221.032439:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205221.032451:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205221.032454:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205221.032460:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205221.032464:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205221.032470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205221.032473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205221.032476:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205221.032480:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205221.096509:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205221.096509:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205221.096521:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205221.096521:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205221.096526:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205221.096526:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205221.096535:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205221.096536:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205221.096542:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205221.096542:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205221.096550:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.096550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205221.096555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205221.096558:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205221.096562:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205221.096563:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205221.096566:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.096571:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205221.096675:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205221.096693:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205221.096696:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205221.096701:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205221.096717:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205221.096726:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6675 00000400:00000200:2.0:1788205221.096736:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205221.096743:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205221.096746:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205221.096750:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.096753:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.096756:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.096758:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205221.096762:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205221.096765:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205221.096768:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205221.096771:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205221.096774:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205221.096777:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205221.096780:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.096782:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.096784:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.096786:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205221.096797:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205221.096801:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205221.096804:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205221.096808:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205221.096811:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205221.096814:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.096817:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205221.096820:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205221.096833:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205221.096837:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205221.096842:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205221.096845:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.096849:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205221.096852:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205221.096858:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205221.096867:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205221.096875:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205221.096892:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205221.096904:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205221.096909:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205221.096915:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205221.096939:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205221.096942:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205221.096946:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205221.096947:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205221.096949:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205221.096952:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205221.096955:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6675 00000400:00000200:2.0:1788205221.096959:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205221.096963:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205221.096966:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205221.096970:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205221.096973:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205221.096973:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.096979:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205221.096980:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205221.096985:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205221.096985:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205221.096990:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205221.096992:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205221.096995:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:0.0:1788205221.096997:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205221.096998:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205221.097003:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205221.097007:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205221.097014:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205221.097017:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205221.097018:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205221.097022:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205221.097023:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097027:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 02000000:00000001:0.0:1788205221.097028:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1788205221.097030:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205221.097033:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097036:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205221.097037:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205221.097042:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205221.097046:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205221.097051:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205940864:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205221.097056:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205221.097057:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205221.097061:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205221.097061:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097064:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1788205221.097066:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.097070:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1788205221.097071:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205221.097076:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205221.097077:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 00000100:00000001:2.0:1788205221.097078:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.097083:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205221.097085:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205221.097089:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000200:0.0:1788205221.097093:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205221.097094:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205221.097098:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097100:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1788205221.097102:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 02000000:00000001:2.0:1788205221.097102:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205221.097106:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205221.097113:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 02000000:00000010:2.0:1788205221.097113:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788205221.097120:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205221.097125:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940864, portal 25 00000100:00100000:2.0:1788205221.097127:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205221.097130:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205221.097133:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788205221.097145:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205221.097145:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205221.097149:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205221.097158:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097161:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205221.097168:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097175:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205221.097177:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205221.097180:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205221.097181:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205221.097183:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:0.0:1788205221.097185:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000001:2.0:1788205221.097187:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205221.097190:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6675 00000100:00000200:0.0:1788205221.097196:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205940864, offset 0 00000400:00000200:2.0:1788205221.097199:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205221.097204:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:0.0:1788205221.097209:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205221.097210:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205221.097216:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097219:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1788205221.097223:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788205221.097223:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097229:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205221.097231:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1788205221.097234:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000020:00000040:2.0:1788205221.097235:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205221.097238:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205221.097242:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000400:00000200:0.0:1788205221.097244:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1788205221.097246:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205221.097250:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205221.097252:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:2.0:1788205221.097256:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205221.097260:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788205221.097261:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.097267:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000200:0.0:1788205221.097268:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097271:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:0.0:1788205221.097277:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59206:59206:256:4294967295] 192.168.201.151@tcp LPNI seq info [59186:59186:8:4294967295] 00000020:00000040:2.0:1788205221.097277:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205221.097283:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:0.0:1788205221.097288:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205221.097288:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205221.097293:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205221.097296:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205221.097301:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205221.097310:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205221.097313:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097317:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:0.0:1788205221.097318:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000008:00100000:2.0:1788205221.097320:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205221.097324:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097326:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788205221.097330:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000010:2.0:1788205221.097332:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000800:00000200:0.0:1788205221.097336:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000020:00000040:2.0:1788205221.097336:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205221.097341:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205221.097344:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788205221.097347:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205221.097348:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205221.097350:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205221.097352:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205221.097355:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c800. 00000100:00000001:2.0:1788205221.097359:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788205221.097389:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000100:00100000:2.0:1788205221.097395:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205221.097404:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205221.097406:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205221.097412:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1788205221.097422:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097422:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1788205221.097428:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000100:00080000:2.0:1788205221.097428:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205221.097435:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205221.097436:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205221.097438:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1788205221.097441:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205221.097442:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205221.097448:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6675 00000400:00000200:0.0:1788205221.097452:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205221.097457:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.097461:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000200:2.0:1788205221.097462:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:0.0:1788205221.097466:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00080000:2.0:1788205221.097467:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205221.097472:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205221.097475:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788205221.097476:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.097480:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000200:0.0:1788205221.097484:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205221.097484:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205221.097487:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205221.097491:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205221.097494:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205221.097498:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205221.097502:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:0.0:1788205221.097504:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788205221.097505:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:0.0:1788205221.097512:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205221.097512:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097518:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205221.097523:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205221.097530:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097530:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205221.097534:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1788205221.097535:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205221.097540:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205221.097542:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.097543:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205221.097545:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1788205221.097550:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000001:2.0:1788205221.097550:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205221.097556:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.097557:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301352450 00000100:00000001:2.0:1788205221.097559:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205221.097562:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205221.097565:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097567:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205221.097573:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205221.097576:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205221.097579:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205221.097581:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205221.097582:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205221.097584:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205221.097586:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205221.097590:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205221.097593:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205221.097599:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205221.097607:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000010:0.0:1788205221.097610:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:2.0:1788205221.097610:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205221.097618:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097621:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205221.097630:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.097632:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301352450 00000100:00000040:2.0:1788205221.097636:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6686) 00000100:00000010:0.0:1788205221.097637:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9540. 00000100:00000001:0.0:1788205221.097643:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205221.097647:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205221.097651:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.097654:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205221.097660:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205221/real 1788205221] req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205221.097670:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205221.097672:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205221.097674:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205221.097677:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205221.097679:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205221.097681:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205221.097685:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205221.097693:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097724:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205221.097726:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097729:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.097731:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205221.097735:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097742:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205221.097747:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097755:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205221.097758:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097759:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.097760:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097763:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205221.097765:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205221.097768:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205221.097772:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205221.097775:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205221.097779:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205221.097788:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205940864:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205221.097793:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097795:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.097798:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205940864/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097805:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205221.097808:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205221.097811:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205221.097814:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205221.097816:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205221.097818:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205221.097820:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205221.097823:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205221.097827:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205221.097830:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205221.097832:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205221.097835:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205221.097837:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097839:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.097857:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205221.097859:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.097861:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205221.097863:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205221.097865:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205221.097868:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097876:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205221.097878:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.097880:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205221.097882:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205221.097885:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205221.097888:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940992:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205221.097891:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205221.097893:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205221.097895:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.097897:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205221.097900:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205221.097902:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205221.097906:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205221.097909:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205221.097912:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205221.097915:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205940992, portal 10 00000100:00000001:0.0:1788205221.097917:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205221.097918:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788205221.097926:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.097933:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205221.097935:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205221.097937:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205221.097939:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205940992, offset 0 00000400:00000200:0.0:1788205221.097943:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205221.097947:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.097951:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205221.097955:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205221.097957:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205221.097960:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205221.097963:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205221.097966:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59207:59207:256:4294967295] 192.168.201.151@tcp LPNI seq info [59187:59187:8:4294967295] 00000400:00000200:0.0:1788205221.097970:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.097976:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205221.097981:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.097984:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205221.097988:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205221.097991:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788205221.097995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205221.097998:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788205221.098001:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.098003:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205221.098006:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205221.098010:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205221.098013:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205221.098015:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205221.098017:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205221.098021:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098034:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098037:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098044:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098046:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205221.098049:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205221.098053:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205221.098056:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205221.098059:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301348507 00000100:00000001:0.0:1788205221.098061:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098063:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301348507 00000100:00000001:0.0:1788205221.098065:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098067:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205221.098069:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098071:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205221.098075:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205221/real 1788205221] req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205221.098082:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205221.098084:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205221.098085:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205221.098087:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205221.098089:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205221.098091:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205221.098094:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205221.098102:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098108:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205221.098110:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098112:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098114:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205221.098117:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098123:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205221.098127:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098134:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205221.098136:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098138:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098139:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098141:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205221.098143:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205221.098146:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205221.098149:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205221.098151:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205221.098155:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205221.098163:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205940992:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205221.098166:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205221.098170:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098176:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205221.098178:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.098180:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205221.098182:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205221.098184:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205221.098187:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941120:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205221.098190:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205221.098192:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205221.098194:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.098195:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205221.098198:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205221.098201:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205221.098204:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205221.098206:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205221.098209:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205221.098212:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941120, portal 4 00000100:00000001:0.0:1788205221.098214:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205221.098215:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788205221.098221:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098228:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205221.098230:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205221.098232:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205221.098234:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205941120, offset 0 00000400:00000200:0.0:1788205221.098238:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205221.098241:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.098245:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205221.098248:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205221.098250:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205221.098252:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205221.098255:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205221.098258:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59208:59208:256:4294967295] 192.168.201.151@tcp LPNI seq info [59188:59188:8:4294967295] 00000400:00000200:0.0:1788205221.098262:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.098267:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205221.098272:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.098275:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205221.098279:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205221.098282:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788205221.098286:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205221.098288:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788205221.098292:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.098294:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205221.098297:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205221.098300:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205221.098303:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205221.098305:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205221.098308:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205221.098311:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098319:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098322:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098328:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098330:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205221.098333:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205221.098336:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205221.098339:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205221.098341:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301348393 00000100:00000001:0.0:1788205221.098343:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098345:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301348393 00000100:00000001:0.0:1788205221.098347:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098349:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205221.098351:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098353:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205221.098356:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205221/real 1788205221] req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205221.098385:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205221.098387:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205221.098389:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205221.098391:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205221.098393:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205221.098395:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205221.098398:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205221.098407:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098413:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205221.098415:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098416:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098418:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205221.098422:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098428:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205221.098432:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098439:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205221.098442:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098443:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098445:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098446:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205221.098449:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205221.098452:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205221.098455:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205221.098457:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205221.098461:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205221.098474:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941120:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205221.098478:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205221.098481:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098488:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205221.098489:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.098492:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205221.098493:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205221.098495:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205221.098498:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205221.098501:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205221.098502:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205221.098504:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205221.098505:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205221.098509:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205221.098511:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205221.098514:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205221.098516:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205221.098519:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205221.098522:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941248, portal 4 00000100:00000001:0.0:1788205221.098524:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205221.098525:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205221.098531:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098537:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205221.098539:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205221.098541:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205221.098543:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205941248, offset 0 00000400:00000200:0.0:1788205221.098547:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205221.098551:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.098555:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205221.098558:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205221.098560:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205221.098562:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205221.098565:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205221.098568:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59209:59209:256:4294967295] 192.168.201.151@tcp LPNI seq info [59189:59189:8:4294967295] 00000400:00000200:0.0:1788205221.098571:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205221.098577:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205221.098580:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.098584:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205221.098587:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205221.098590:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788205221.098594:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205221.098597:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788205221.098600:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205221.098602:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205221.098605:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205221.098609:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205221.098611:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205221.098613:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205221.098615:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205221.098619:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098626:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098628:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098634:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098636:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205221.098639:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205221.098641:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205221.098644:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205221.098647:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301348400 00000100:00000001:0.0:1788205221.098649:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098651:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301348400 00000100:00000001:0.0:1788205221.098654:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301352450 00000100:00000001:0.0:1788205221.098656:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205221.098658:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098659:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205221.098663:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205221/real 1788205221] req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205221.098670:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205221.098671:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205221.098673:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205221.098675:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205221.098677:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205221.098679:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205221.098682:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205221.098691:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098697:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205221.098699:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098700:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098701:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205221.098717:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098724:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205221.098728:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098734:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205221.098738:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098740:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205221.098741:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098743:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205221.098744:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205221.098747:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205221.098750:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205221.098753:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205221.098756:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205221.098764:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941248:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205221.098768:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098770:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098773:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205940992/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098780:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205221.098781:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205221.098783:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205221.098785:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205221.098787:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205221.098788:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205221.098790:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205221.098792:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:0.0:1788205221.098795:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205221.098796:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205221.098797:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205221.098801:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205221.098802:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098804:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098807:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205941120/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098819:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205221.098821:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205221.098822:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205221.098824:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205221.098825:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205221.098827:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205221.098829:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205221.098830:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c800. 02000000:00000001:0.0:1788205221.098832:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205221.098833:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205221.098835:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205221.098837:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205221.098838:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098840:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205221.098843:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205941248/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205221.098849:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205221.098850:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205221.098851:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205221.098854:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205221.098855:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205221.098856:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205221.098858:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205221.098860:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205221.098862:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205221.098863:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205221.098864:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205221.098866:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205221.098867:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098869:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205221.098873:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205221.098874:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205221.098877:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205221.098879:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205221.288561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288569:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205221.288573:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205221.288577:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205221.288581:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288585:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205221.288588:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288590:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288593:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205221.288598:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205221.288602:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205221.288613:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205221.288671:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205221.288683:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205221.288687:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205221.608590:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205221.608598:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205221.608600:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205221.608605:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205221.608608:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205221.608612:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.608614:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205221.608617:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.608619:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.056445:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205222.056455:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205222.056457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.056461:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205222.056464:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205222.056468:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.056470:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205222.056472:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.056475:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.120497:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205222.120506:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205222.120508:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.120513:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205222.120517:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205222.120522:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.120525:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205222.120527:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.120529:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.120727:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205222.120731:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205222.120733:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.120736:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205222.120739:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205222.120741:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.120743:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.120745:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.120748:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205222.312490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205222.312497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205222.312502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205222.312504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205222.312507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205222.312508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205222.312512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205222.312513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205222.313803:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205222.313813:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205222.313819:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205222.632509:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205222.632519:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205222.632522:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.632529:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205222.632534:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205222.632539:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.632543:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.632546:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.632550:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205223.080496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205223.080504:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205223.080506:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205223.080511:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205223.080514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205223.080518:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.080521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205223.080523:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.080526:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205223.144529:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205223.144536:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205223.144538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205223.144544:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205223.144548:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205223.144552:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.144555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205223.144557:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.144560:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205223.146866:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205223.146874:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205223.146877:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205223.146883:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205223.146887:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205223.146891:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205223.146894:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205223.146897:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205223.146901:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205223.336580:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205223.336583:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205223.336587:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336593:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205223.336595:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336597:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336600:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336605:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205223.336607:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336610:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205223.336612:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205223.336613:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205223.336614:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336616:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205223.336617:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336619:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336620:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336623:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205223.336625:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205223.336626:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205223.336627:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336629:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205223.336630:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336632:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205223.336634:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205223.336854:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205223.336858:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205223.336861:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205223.656530:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205223.656540:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205223.656543:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205223.656551:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205223.656555:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205223.656560:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205223.656564:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205223.656567:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205223.656571:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205224.104491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205224.104501:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205224.104503:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205224.104510:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205224.104514:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205224.104519:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205224.104521:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205224.104524:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205224.104528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205224.168490:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205224.168500:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205224.168502:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205224.168509:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205224.168513:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205224.168517:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205224.168521:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205224.168524:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205224.168528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205224.168617:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205224.168621:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205224.168622:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205224.168625:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205224.168627:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205224.168629:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.168630:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205224.168632:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.168634:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205224.361811:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205224.361820:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205224.361829:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205224.361830:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205224.361834:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205224.361835:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205224.361839:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205224.361840:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205224.361997:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205224.362001:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205224.362005:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205224.680466:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205224.680477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205224.680480:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205224.680487:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205224.680491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205224.680497:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.680500:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205224.680503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.680506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205225.128499:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205225.128507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205225.128509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205225.128513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205225.128517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205225.128521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205225.128523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205225.128525:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205225.128528:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205225.192477:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205225.192486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205225.192488:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205225.192495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205225.192499:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205225.192504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205225.192507:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205225.192510:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205225.192513:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205225.193342:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205225.193347:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205225.193348:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205225.193352:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205225.193354:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205225.193356:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.193358:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205225.193360:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.193376:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205225.386435:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205225.386446:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205225.386453:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205225.386454:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205225.386458:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205225.386459:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205225.386463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205225.386464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205225.387130:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205225.387135:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205225.387139:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205225.704478:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301352450, at 4301352960 00000100:00000040:1.0:1788205225.704483:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301352450, at 4301352960 00000100:00000010:0.0:1788205225.704492:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000010:1.0:1788205225.704494:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9540. 00000100:00000001:3.0:1788205225.704770:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205225.704774:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205225.704776:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205225.704780:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205225.704782:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205225.704786:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.704788:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205225.704791:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.704794:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205226.152472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205226.152478:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205226.152481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205226.152486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205226.152489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205226.152493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205226.152495:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205226.152498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205226.152501:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205226.216460:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205226.216460:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205226.216465:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205226.216467:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205226.216468:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205226.216469:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205226.216471:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00080000:2.0:1788205226.216472:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205226.216475:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205226.216475:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205226.216478:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.216480:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205226.216481:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6681 00000100:00000001:3.0:1788205226.216482:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.216485:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205226.216488:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205226.216493:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205226.216497:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205226.216500:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216503:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216505:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216508:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205226.216512:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205226.216514:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205226.216516:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205226.216519:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205226.216522:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205226.216524:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205226.216527:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216529:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216530:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216532:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205226.216535:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205226.216538:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216540:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205226.216543:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205226.216546:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205226.216549:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216551:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205226.216553:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205226.216561:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000010:2.0:1788205226.216562:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000100:00000001:1.0:1788205226.216563:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205226.216564:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205226.216565:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 00000100:00100000:1.0:1788205226.216567:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205226.216568:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788205226.216568:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205226.216570:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205226.216570:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205226.216573:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205226.216573:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205226.216574:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205226.216575:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205226.216576:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205226.216580:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205226.216586:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788205226.216592:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205226.216601:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205226.216611:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205226.216624:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205226.216630:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205226.216664:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205226.216668:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205226.216671:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205226.216672:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205226.216674:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205226.216677:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205226.216679:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6681 00000400:00000200:2.0:1788205226.216683:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205226.216686:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205226.216688:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205226.216690:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216692:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216694:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216696:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205226.216698:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205226.216699:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205226.216701:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205226.216703:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205226.216705:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205226.216707:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205226.216710:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216712:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216713:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216715:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205226.216717:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205226.216720:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216722:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205226.216724:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205226.216728:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205226.216730:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216732:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205226.216735:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205226.216738:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216740:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205226.216743:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205226.216745:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205226.216748:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205226.216749:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216750:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205226.216752:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205226.216753:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205226.216756:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205226.216759:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205226.216763:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205226.216770:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205226.216772:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205226.216775:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205226.216781:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205226.216783:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205226.216786:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205226.216787:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205226.216789:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205226.216791:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205226.216793:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6681 00000400:00000200:2.0:1788205226.216795:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205226.216798:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205226.216800:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205226.216802:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216804:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216806:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216807:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205226.216809:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205226.216810:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205226.216811:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205226.216813:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205226.216814:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205226.216816:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205226.216818:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216820:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216821:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216822:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205226.216824:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205226.216826:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216828:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205226.216829:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205226.216831:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205226.216833:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216834:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205226.216836:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205226.216838:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216839:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205226.216841:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205226.216843:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205226.216845:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205226.216846:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216856:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205226.216858:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205226.216859:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205226.216863:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000100:00000001:2.0:1788205226.216866:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205226.216870:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205226.216876:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205226.216878:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205226.216881:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205226.216887:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205226.216888:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205226.216891:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205226.216892:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205226.216893:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205226.216895:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205226.216897:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6681 00000400:00000200:2.0:1788205226.216900:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205226.216902:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205226.216904:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205226.216906:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216908:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216910:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216912:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205226.216913:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205226.216914:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205226.216915:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205226.216917:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205226.216918:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205226.216920:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205226.216922:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216924:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205226.216926:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216928:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205226.216930:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205226.216932:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205226.216934:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205226.216935:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205226.216937:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205226.216938:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216940:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205226.216941:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205226.216943:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216944:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205226.216947:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205226.216956:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205226.216958:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205226.216959:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205226.216960:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205226.216961:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205226.216963:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205226.216966:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205226.216968:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205226.216973:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205226.216979:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205226.216986:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205226.216989:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205226.216997:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205226.217001:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6691) 00000100:00000001:0.0:1788205226.217113:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205226.217117:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205226.217119:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205226.217125:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217132:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205226.217133:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217137:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205226.217139:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205226.217141:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205226.217144:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205941376:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205226.217148:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205226.217152:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205226.217155:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217157:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205226.217162:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205226.217163:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205226.217168:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205226.217173:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205226.217179:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205226.217185:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941376, portal 25 00000100:00000001:0.0:1788205226.217188:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205226.217189:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:0.0:1788205226.217194:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217202:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205226.217205:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205226.217208:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205226.217210:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205941376, offset 0 00000400:00000200:0.0:1788205226.217217:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217224:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217230:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205226.217235:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205226.217239:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205226.217243:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205226.217246:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217249:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59210:59210:256:4294967295] 192.168.201.151@tcp LPNI seq info [59190:59190:8:4294967295] 00000400:00000200:0.0:1788205226.217253:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217262:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205226.217267:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217271:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205226.217278:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205226.217293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788205226.217299:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205226.217303:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788205226.217308:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217310:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205226.217318:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205226.217322:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205226.217326:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205226.217329:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205226.217332:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205226.217336:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217344:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.217348:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217359:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.217362:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205226.217393:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205226.217397:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205226.217400:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301357570 00000100:00000010:0.0:1788205226.217407:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:0.0:1788205226.217413:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205226.217418:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301357570 00000100:00000010:0.0:1788205226.217421:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9ea0. 00000100:00000001:0.0:1788205226.217423:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205226.217426:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205226.217429:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.217431:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205226.217436:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205226/real 1788205226] req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205226.217444:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205226.217446:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205226.217448:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205226.217451:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205226.217452:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205226.217455:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205226.217458:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205226.217466:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217472:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205226.217473:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217475:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.217477:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205226.217481:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217488:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205226.217493:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217499:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205226.217502:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217504:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.217505:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217507:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205226.217508:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205226.217511:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205226.217514:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205226.217517:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205226.217521:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205226.217529:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205941376:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205226.217533:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205226.217536:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217542:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205226.217543:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217545:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205226.217547:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205226.217549:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205226.217552:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941504:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205226.217555:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205226.217557:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205226.217558:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217560:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205226.217563:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205226.217565:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205226.217568:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205226.217571:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205226.217574:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205226.217577:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941504, portal 10 00000100:00000001:0.0:1788205226.217578:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205226.217579:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:0.0:1788205226.217585:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217592:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205226.217594:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205226.217596:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205226.217598:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205941504, offset 0 00000400:00000200:0.0:1788205226.217602:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217605:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217609:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205226.217611:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205226.217614:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205226.217627:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205226.217629:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217632:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59211:59211:256:4294967295] 192.168.201.151@tcp LPNI seq info [59191:59191:8:4294967295] 00000400:00000200:0.0:1788205226.217636:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217651:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205226.217656:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217659:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205226.217663:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205226.217666:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788205226.217670:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205226.217673:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788205226.217677:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217678:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205226.217682:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205226.217686:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205226.217689:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205226.217691:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205226.217693:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205226.217696:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217703:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.217706:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217712:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.217713:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205226.217717:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205226.217721:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205226.217723:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205226.217725:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301353627 00000100:00000001:0.0:1788205226.217727:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.217730:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301353627 00000100:00000001:0.0:1788205226.217732:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.217734:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205226.217735:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.217737:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205226.217741:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205226/real 1788205226] req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205226.217748:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205226.217749:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205226.217751:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205226.217753:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205226.217755:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205226.217757:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205226.217760:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205226.217768:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217775:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205226.217776:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217778:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.217779:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205226.217782:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217789:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205226.217792:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217798:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205226.217800:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217802:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.217803:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.217804:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205226.217806:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205226.217808:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205226.217811:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205226.217813:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205226.217816:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205226.217824:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941504:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205226.217827:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205226.217830:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217842:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205226.217843:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217845:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205226.217847:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205226.217849:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205226.217851:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941632:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205226.217854:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205226.217855:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205226.217857:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.217858:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205226.217861:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205226.217863:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205226.217866:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205226.217868:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205226.217871:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205226.217873:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941632, portal 4 00000100:00000001:0.0:1788205226.217875:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205226.217876:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:0.0:1788205226.217881:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217887:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205226.217889:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205226.217891:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205226.217893:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205941632, offset 0 00000400:00000200:0.0:1788205226.217896:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217899:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217902:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205226.217905:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205226.217907:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205226.217909:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205226.217911:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205226.217915:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59212:59212:256:4294967295] 192.168.201.151@tcp LPNI seq info [59192:59192:8:4294967295] 00000400:00000200:0.0:1788205226.217918:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.217925:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205226.217929:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217932:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205226.217936:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205226.217938:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788205226.217942:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205226.217944:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788205226.217948:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.217950:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205226.217953:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205226.217956:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205226.217959:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205226.217960:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205226.217962:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205226.217965:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217972:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.217975:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.217981:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.217982:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205226.217985:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205226.217987:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205226.217989:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205226.217991:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301353513 00000100:00000001:0.0:1788205226.217994:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.217996:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301353513 00000100:00000001:0.0:1788205226.217998:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.217999:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205226.218001:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.218003:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205226.218006:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205226/real 1788205226] req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205226.218014:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205226.218015:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205226.218017:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205226.218019:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205226.218021:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205226.218023:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205226.218025:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205226.218034:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218040:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205226.218041:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218043:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.218044:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205226.218047:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218053:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205226.218057:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218064:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205226.218066:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218067:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.218069:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218070:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205226.218072:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205226.218075:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205226.218078:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205226.218080:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205226.218084:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205226.218091:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941632:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205226.218095:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205226.218098:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218104:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205226.218105:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.218107:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205226.218109:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205226.218110:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205226.218113:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205226.218115:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205226.218117:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205226.218119:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205226.218120:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205226.218123:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205226.218124:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205226.218127:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205226.218129:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205226.218131:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205226.218134:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941760, portal 4 00000100:00000001:0.0:1788205226.218135:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205226.218136:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205226.218141:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218147:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205226.218149:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205226.218151:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205226.218154:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205941760, offset 0 00000400:00000200:0.0:1788205226.218157:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205226.218160:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.218164:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205226.218166:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205226.218168:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205226.218170:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205226.218172:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205226.218174:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59213:59213:256:4294967295] 192.168.201.151@tcp LPNI seq info [59193:59193:8:4294967295] 00000400:00000200:0.0:1788205226.218178:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205226.218183:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205226.218188:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.218191:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205226.218195:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205226.218197:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788205226.218201:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205226.218203:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788205226.218206:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205226.218208:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205226.218211:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205226.218214:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205226.218216:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205226.218218:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205226.218220:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205226.218224:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218230:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.218233:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218239:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.218241:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205226.218244:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205226.218247:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205226.218251:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205226.218253:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301353519 00000100:00000001:0.0:1788205226.218255:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.218256:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301353519 00000100:00000001:0.0:1788205226.218258:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301357570 00000100:00000001:0.0:1788205226.218260:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205226.218262:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.218264:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205226.218267:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205226/real 1788205226] req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205226.218275:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205226.218276:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205226.218278:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205226.218298:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205226.218300:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205226.218302:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205226.218305:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205226.218312:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218318:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205226.218319:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218321:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.218322:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205226.218325:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218331:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205226.218334:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218339:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205226.218342:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218343:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205226.218344:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218346:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205226.218348:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205226.218350:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205226.218352:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205226.218354:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205226.218358:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205226.218384:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205941760:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205226.218387:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218390:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.218393:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205941376/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218400:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205226.218402:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205226.218404:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205226.218407:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205226.218409:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205226.218410:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205226.218412:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205226.218414:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:0.0:1788205226.218418:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205226.218420:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205226.218422:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:0.0:1788205226.218425:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205226.218426:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218428:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.218431:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205941504/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218437:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205226.218438:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205226.218440:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205226.218442:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205226.218443:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205226.218444:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205226.218446:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205226.218447:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:0.0:1788205226.218449:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205226.218450:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205226.218451:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:0.0:1788205226.218453:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205226.218454:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218455:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.218458:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205941632/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218464:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205226.218465:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205226.218466:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205226.218468:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205226.218469:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205226.218470:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205226.218471:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205226.218472:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:0.0:1788205226.218474:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205226.218476:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205226.218477:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:0.0:1788205226.218479:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205226.218480:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218481:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205226.218484:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205941760/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205226.218496:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205226.218497:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205226.218498:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205226.218501:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205226.218502:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205226.218503:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205226.218505:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205226.218506:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:0.0:1788205226.218507:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205226.218509:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205226.218510:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205226.218511:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205226.218512:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218514:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205226.218521:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205226.218522:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205226.218524:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205226.218526:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205226.409583:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205226.409589:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205226.409594:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205226.409596:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205226.409599:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205226.409600:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205226.409604:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205226.409605:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205226.409727:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205226.409730:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205226.409733:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205226.728550:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205226.728557:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205226.728559:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205226.728564:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205226.728568:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205226.728572:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.728574:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205226.728577:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.729145:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.176476:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205227.176482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205227.176484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.176489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205227.176492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205227.176495:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205227.176498:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205227.176500:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205227.176502:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.240480:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205227.240486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205227.240488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.240492:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205227.240494:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205227.240497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.240499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.240501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.240502:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.240567:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205227.240569:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205227.240570:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.240572:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205227.240573:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205227.240574:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205227.240576:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205227.240577:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205227.240578:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205227.432504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205227.432512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205227.432518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205227.432519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205227.432523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205227.432524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205227.432527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205227.432528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205227.433407:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205227.433411:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205227.433414:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205227.752469:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205227.752477:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205227.752479:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.752485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205227.752488:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205227.752492:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.752495:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.752498:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.752501:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205228.200532:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205228.200540:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205228.200542:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205228.200548:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205228.200551:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205228.200555:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.200557:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205228.200560:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.200562:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205228.264496:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205228.264504:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205228.264507:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205228.264512:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205228.264516:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205228.264519:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.264522:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205228.264525:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.264528:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205228.264632:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205228.264636:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205228.264637:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205228.264639:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205228.264641:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205228.264644:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205228.264646:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205228.264647:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205228.264649:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205228.457246:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205228.457253:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205228.457257:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205228.457259:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205228.457262:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205228.457263:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205228.457266:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205228.457267:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205228.457328:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205228.457332:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205228.457335:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205228.777800:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205228.777805:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205228.777807:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205228.777810:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205228.777813:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205228.777815:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205228.777818:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205228.777820:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205228.777821:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205229.224670:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205229.224677:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205229.224680:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205229.224685:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205229.224689:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205229.224692:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205229.224694:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205229.224696:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205229.224698:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205229.288493:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205229.288501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205229.288503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205229.288508:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205229.288511:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205229.288516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205229.288518:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205229.288521:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205229.288524:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205229.288691:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205229.288693:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205229.288694:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205229.288696:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205229.288698:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205229.288699:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.288701:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205229.288701:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.288703:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205229.480612:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205229.480619:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205229.480625:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205229.480626:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205229.480629:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205229.480631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205229.480633:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205229.480635:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205229.480683:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205229.480686:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205229.480688:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205229.800798:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205229.800804:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205229.800806:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205229.800809:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205229.800812:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205229.800815:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.800817:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205229.800819:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.800821:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205230.248779:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205230.248786:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205230.248789:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205230.248794:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205230.248798:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205230.248802:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205230.248805:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205230.248807:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205230.248810:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205230.312475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205230.312482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205230.312484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205230.312489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205230.312493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205230.312496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.312499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205230.312501:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.312504:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205230.312605:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205230.312607:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205230.312609:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205230.312611:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205230.312614:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205230.312616:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205230.312622:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205230.312624:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205230.312626:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205230.504709:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205230.504714:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205230.504718:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205230.504719:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205230.504722:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205230.504722:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205230.504724:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205230.504725:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205230.504775:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205230.504779:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205230.504782:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205230.824468:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301357570, at 4301358080 00000100:00000001:3.0:1788205230.824470:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205230.824476:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205230.824481:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205230.824484:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00100000:3.0:1788205230.824485:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205230.824491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205230.824494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.824497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205230.824500:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.824503:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205230.824604:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301357570, at 4301358080 00000100:00000010:1.0:1788205230.824611:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9ea0. 00000100:00000001:1.0:1788205231.272635:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205231.272646:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205231.272649:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205231.272656:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205231.272660:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205231.272665:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205231.272668:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205231.272671:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205231.272675:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205231.336526:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205231.336546:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205231.336548:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205231.336556:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205231.336560:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205231.336566:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205231.336569:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205231.336573:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205231.336577:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205231.336609:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205231.336614:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205231.336617:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205231.336621:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205231.336624:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205231.336627:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205231.336630:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205231.336632:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205231.336635:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205231.339403:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205231.339415:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205231.339417:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205231.339421:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205231.339424:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205231.339432:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6686 00000400:00000200:2.0:1788205231.339442:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205231.339447:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205231.339450:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205231.339453:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339457:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339459:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339461:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205231.339466:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205231.339467:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205231.339470:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205231.339473:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205231.339476:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205231.339478:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205231.339480:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339482:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339484:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339485:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205231.339488:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205231.339491:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339494:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205231.339496:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205231.339500:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205231.339503:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339505:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205231.339507:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205231.339517:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205231.339520:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205231.339524:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205231.339526:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339539:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205231.339542:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205231.339546:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205231.339554:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0f400. 00000100:00000001:2.0:1788205231.339559:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205231.339568:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205231.339576:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205231.339580:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205231.339584:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205231.339614:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205231.339617:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205231.339620:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205231.339621:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205231.339623:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205231.339626:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205231.339629:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6686 00000400:00000200:2.0:1788205231.339633:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205231.339636:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205231.339638:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205231.339649:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339652:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339654:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339656:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205231.339658:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205231.339659:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205231.339661:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205231.339663:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205231.339665:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205231.339667:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205231.339669:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339671:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339673:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339675:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205231.339677:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205231.339680:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339682:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205231.339684:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205231.339688:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205231.339690:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339692:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205231.339696:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205231.339699:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339701:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205231.339705:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205231.339707:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205231.339709:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205231.339710:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339712:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205231.339713:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205231.339715:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205231.339718:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788205231.339722:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205231.339727:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205231.339734:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205231.339736:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205231.339739:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205231.339745:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205231.339748:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205231.339751:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205231.339752:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205231.339753:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205231.339756:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205231.339758:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6686 00000400:00000200:2.0:1788205231.339761:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205231.339764:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205231.339766:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205231.339768:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339770:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339772:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339773:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205231.339775:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205231.339776:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205231.339778:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205231.339780:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205231.339781:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205231.339783:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205231.339785:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339787:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339789:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339791:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205231.339793:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205231.339795:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339797:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205231.339798:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205231.339801:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205231.339802:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339804:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205231.339807:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205231.339809:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339810:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205231.339814:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205231.339816:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205231.339818:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205231.339819:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339820:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205231.339821:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205231.339823:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205231.339826:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205231.339829:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205231.339833:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205231.339840:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205231.339842:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205231.339845:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205231.339863:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205231.339866:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205231.339869:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205231.339871:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205231.339872:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205231.339875:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205231.339877:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6686 00000400:00000200:2.0:1788205231.339880:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205231.339883:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205231.339885:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205231.339887:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339889:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339954:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339956:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205231.339958:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205231.339959:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205231.339961:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205231.339963:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205231.339965:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205231.339967:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205231.339970:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339971:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205231.339973:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339975:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205231.339977:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205231.339979:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205231.339981:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205231.339983:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205231.339986:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205231.339988:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339989:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205231.339991:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205231.339994:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.339995:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205231.340000:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205231.340003:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205231.340005:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205231.340006:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.340008:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205231.340009:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205231.340011:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205231.340015:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205231.340018:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205231.340245:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205231.340254:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205231.340256:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205231.340260:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205231.340268:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205231.340272:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6696) 00000100:00000001:0.0:1788205231.341001:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205231.341006:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205231.341009:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205231.341016:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341135:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205231.341137:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341141:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205231.341144:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205231.341147:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205231.341150:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205941888:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205231.341163:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205231.341167:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205231.341170:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341172:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205231.341178:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205231.341180:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205231.341185:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205231.341189:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205231.341195:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205231.341202:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205941888, portal 25 00000100:00000001:0.0:1788205231.341204:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205231.341205:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:0.0:1788205231.341210:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341217:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205231.341219:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205231.341221:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205231.341224:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205941888, offset 0 00000400:00000200:0.0:1788205231.341230:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205231.341238:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.341244:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205231.341249:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205231.341253:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205231.341257:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205231.341260:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205231.341263:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59214:59214:256:4294967295] 192.168.201.151@tcp LPNI seq info [59194:59194:8:4294967295] 00000400:00000200:0.0:1788205231.341267:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.341275:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205231.341280:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.341286:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205231.341292:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205231.341296:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205231.341300:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205231.341303:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205231.341307:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.341309:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205231.341316:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205231.341320:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205231.341325:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205231.341327:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205231.341330:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205231.341334:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341343:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.341347:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341352:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.341354:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205231.341360:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205231.341391:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205231.341394:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301362694 00000100:00000010:0.0:1788205231.341399:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:0.0:1788205231.341405:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205231.341411:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301362694 00000100:00000010:0.0:1788205231.341414:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9c00. 00000100:00000001:0.0:1788205231.341417:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205231.341420:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205231.341422:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.341425:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205231.341429:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205231/real 1788205231] req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205231.341445:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205231.341447:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205231.341449:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205231.341451:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205231.341453:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205231.341456:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205231.341459:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205231.341467:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341473:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205231.341474:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341476:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.341478:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205231.341481:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341487:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205231.341492:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341498:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205231.341501:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341503:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.341504:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341506:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205231.341508:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205231.341511:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205231.341515:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205231.341517:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205231.341521:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205231.341540:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205941888:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205231.341546:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205231.341549:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341554:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205231.341555:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341557:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205231.341559:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205231.341561:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205231.341564:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942016:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205231.341567:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205231.341568:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205231.341570:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341571:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205231.341574:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205231.341576:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205231.341579:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205231.341582:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205231.341585:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205231.341587:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942016, portal 10 00000100:00000001:0.0:1788205231.341589:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205231.341590:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:0.0:1788205231.341596:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341601:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205231.341603:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205231.341605:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205231.341607:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205942016, offset 0 00000400:00000200:0.0:1788205231.341611:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205231.341615:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.341618:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205231.341621:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205231.341623:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205231.341625:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205231.341627:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205231.341630:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59215:59215:256:4294967295] 192.168.201.151@tcp LPNI seq info [59195:59195:8:4294967295] 00000400:00000200:0.0:1788205231.341634:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.341639:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205231.341654:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.341657:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205231.341661:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205231.341663:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205231.341667:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205231.341669:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205231.341673:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.341674:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205231.341677:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205231.341681:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205231.341683:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205231.341685:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205231.341688:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205231.341691:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341698:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.341701:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341707:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.341709:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205231.341711:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205231.341715:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205231.341718:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205231.341720:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301358751 00000100:00000001:0.0:1788205231.341722:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.341724:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301358751 00000100:00000001:0.0:1788205231.341726:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.341728:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205231.341730:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.341732:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205231.341735:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205231/real 1788205231] req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205231.341742:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205231.341743:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205231.341745:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205231.341747:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205231.341749:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205231.341750:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205231.341754:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205231.341761:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341767:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205231.341769:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341771:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.341772:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205231.341775:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341781:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205231.341784:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341790:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205231.341793:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341794:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.341795:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.341797:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205231.341800:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205231.341802:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205231.341806:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205231.341807:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205231.341810:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205231.341818:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942016:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205231.341822:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205231.341826:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.341832:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205231.341834:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341836:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205231.341837:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205231.341840:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205231.341842:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942144:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205231.341845:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205231.341847:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205231.341849:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.341851:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205231.341865:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205231.341867:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205231.341871:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205231.341873:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205231.341875:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205231.341878:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942144, portal 4 00000100:00000001:0.0:1788205231.341880:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205231.341882:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:0.0:1788205231.341887:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.344685:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205231.344694:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205231.344704:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205231.344712:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205942144, offset 0 00000400:00000200:0.0:1788205231.344718:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205231.344727:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.344735:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205231.344740:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205231.344744:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205231.344747:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205231.344750:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205231.344753:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59216:59216:256:4294967295] 192.168.201.151@tcp LPNI seq info [59196:59196:8:4294967295] 00000400:00000200:0.0:1788205231.344758:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.344766:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205231.344772:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.344778:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205231.344784:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205231.344788:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205231.344793:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205231.344796:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205231.344800:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.344802:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205231.344811:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205231.344815:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205231.344819:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205231.344821:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205231.344825:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205231.344832:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.345730:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.345741:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.345750:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.345751:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205231.345760:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205231.345764:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205231.345769:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205231.345772:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301358641 00000100:00000001:0.0:1788205231.345776:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.345778:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301358641 00000100:00000001:0.0:1788205231.345780:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.345782:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205231.345785:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.345787:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205231.345791:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205231/real 1788205231] req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205231.345801:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205231.345802:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205231.345805:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205231.345808:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205231.345809:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205231.345812:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205231.345815:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205231.345822:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.345828:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205231.345830:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.345832:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.345834:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205231.345837:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.345843:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205231.345847:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.347435:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205231.347440:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.347442:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.347444:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.347447:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205231.347450:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205231.347453:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205231.347461:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205231.347464:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205231.347470:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205231.347480:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942144:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205231.347486:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205231.347490:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.347497:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205231.347499:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.347503:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205231.347506:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205231.347509:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205231.347512:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205231.347516:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205231.347519:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205231.347522:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205231.347524:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205231.347543:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205231.347546:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205231.347551:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205231.347555:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205231.347561:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205231.347567:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942272, portal 4 00000100:00000001:0.0:1788205231.347569:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205231.347570:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:0.0:1788205231.347578:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.347584:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205231.347586:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205231.347589:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205231.347592:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205942272, offset 0 00000400:00000200:0.0:1788205231.347597:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205231.347606:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.347612:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205231.347618:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205231.347622:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205231.347625:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205231.347627:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205231.347631:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59217:59217:256:4294967295] 192.168.201.151@tcp LPNI seq info [59197:59197:8:4294967295] 00000400:00000200:0.0:1788205231.347635:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205231.348308:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205231.348316:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.348322:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205231.348329:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205231.348332:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788205231.348338:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205231.348341:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788205231.348345:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205231.348348:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:0.0:1788205231.348356:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205231.348360:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205231.348391:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:0.0:1788205231.348393:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:0.0:1788205231.348398:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205231.348402:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348411:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.348415:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348421:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.348423:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205231.348427:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205231.348431:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205231.348435:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205231.348438:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301358650 00000100:00000001:0.0:1788205231.348441:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.348443:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301358650 00000100:00000001:0.0:1788205231.348445:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301362694 00000100:00000001:0.0:1788205231.348447:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205231.348450:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.348452:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205231.348456:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205231/real 1788205231] req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205231.348464:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205231.348465:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205231.348467:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:0.0:1788205231.348470:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:0.0:1788205231.348472:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:0.0:1788205231.348474:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205231.348477:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205231.348484:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348491:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205231.348493:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348495:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.348496:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205231.348500:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348507:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205231.348511:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348518:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205231.348520:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348522:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205231.348523:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348525:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205231.348538:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205231.348542:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205231.348545:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205231.348548:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205231.348552:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205231.348565:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942272:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205231.348570:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348573:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.348577:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205941888/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348583:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205231.348586:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205231.348588:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205231.348592:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205231.348595:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205231.348597:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205231.348599:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205231.348601:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0f400. 02000000:00000001:0.0:1788205231.348605:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205231.348607:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205231.348609:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:0.0:1788205231.348612:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205231.348614:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348615:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.348620:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205942016/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348627:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205231.348628:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205231.348630:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205231.348632:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205231.348633:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205231.348635:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205231.348637:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205231.348638:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 02000000:00000001:0.0:1788205231.348640:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205231.348652:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205231.348654:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:0.0:1788205231.348657:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205231.348658:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348660:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.348664:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205942144/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348670:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205231.348672:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205231.348673:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205231.348675:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205231.348677:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205231.348678:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205231.348680:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205231.348681:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:0.0:1788205231.348684:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205231.348685:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205231.348686:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:0.0:1788205231.348688:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205231.348690:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348691:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205231.348694:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205942272/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205231.348700:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205231.348701:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205231.348703:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205231.348705:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205231.348706:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205231.348707:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205231.348709:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205231.348710:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:0.0:1788205231.348712:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205231.348713:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205231.348715:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:0.0:1788205231.348717:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205231.348718:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348720:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205231.348728:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205231.348730:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205231.348733:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205231.348735:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530440:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205231.530450:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530458:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205231.530461:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205231.530465:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205231.530470:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530473:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205231.530476:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530478:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530481:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205231.530486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205231.530490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205231.530492:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205231.530722:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205231.530727:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205231.530731:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205231.848624:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205231.848634:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205231.848637:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205231.848644:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205231.848649:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205231.848654:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205231.848658:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205231.848662:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205231.848665:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.296668:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205232.296679:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.296682:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.296689:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205232.296693:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.296698:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.296701:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.296705:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.296709:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.360640:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205232.360650:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205232.360653:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.360659:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205232.360663:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205232.360668:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.360672:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205232.360675:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.360679:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.360780:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205232.360783:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.360784:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.360787:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205232.360789:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.360792:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.360794:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.360795:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.360797:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205232.554010:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205232.554021:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205232.554029:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205232.554030:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205232.554034:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205232.554036:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205232.554039:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205232.554041:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205232.554316:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205232.554320:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205232.554324:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205232.872530:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205232.872541:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205232.872543:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.872550:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205232.872555:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205232.872560:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.872564:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205232.872567:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.872571:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205233.321717:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205233.321727:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205233.321730:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205233.321737:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205233.321741:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205233.321746:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.321749:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205233.321752:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.321756:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205233.384526:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205233.384537:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205233.384540:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205233.384547:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205233.384554:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205233.384556:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205233.384560:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.384561:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205233.384565:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205233.384566:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1788205233.384571:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205233.384571:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.384576:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205233.384578:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205233.384579:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.384584:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205233.384587:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.384590:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205233.576598:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205233.576602:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205233.576607:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576614:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205233.576617:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576619:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576622:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576629:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205233.576631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576634:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205233.576636:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205233.576637:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205233.576639:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576640:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205233.576642:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576643:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576645:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576649:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205233.576651:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205233.576652:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205233.576653:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576655:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205233.576657:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576658:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205233.576660:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205233.576777:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205233.576782:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205233.576786:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205233.896553:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205233.896564:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205233.896566:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205233.896574:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205233.896578:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205233.896582:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205233.896586:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205233.896589:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205233.896592:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205234.344856:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205234.344864:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205234.344866:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205234.344871:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205234.344874:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205234.344878:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.344880:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205234.344882:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.344885:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205234.408596:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205234.408603:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205234.408605:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205234.408611:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205234.408614:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205234.408618:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205234.408620:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205234.408623:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205234.408625:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205234.408866:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205234.408874:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205234.408876:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205234.408882:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205234.408885:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205234.408889:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.408892:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205234.408895:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.408909:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205234.600538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205234.600547:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205234.600555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205234.600557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205234.600562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205234.600563:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205234.600566:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205234.600568:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205234.600747:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205234.600750:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205234.600754:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205234.920450:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205234.920457:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205234.920459:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205234.920500:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205234.920504:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205234.920509:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205234.920511:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205234.920514:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205234.920517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205235.368531:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205235.368541:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205235.368543:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205235.368551:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205235.368555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205235.368560:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205235.368563:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205235.368566:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205235.368570:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205235.432499:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205235.432506:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205235.432509:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205235.432514:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205235.432518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205235.432521:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.432524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205235.432526:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.432529:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205235.434407:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205235.434416:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205235.434419:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205235.434426:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205235.434429:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205235.434433:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205235.434436:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205235.434439:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205235.434441:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205235.624582:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205235.624594:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205235.624601:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205235.624602:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205235.624606:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205235.624607:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205235.624611:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205235.624612:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205235.624688:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205235.624694:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205235.624698:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205235.944483:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301362694, at 4301363200 00000100:00000001:3.0:1788205235.944485:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205235.944491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205235.944495:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205235.944496:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9c00. 00000100:00100000:3.0:1788205235.944499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205235.944502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205235.944505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.944507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205235.944509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.944510:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205235.944580:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301362694, at 4301363200 00000100:00000010:0.0:1788205235.944587:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000001:1.0:1788205236.392504:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205236.392511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205236.392514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205236.392519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205236.392523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205236.392527:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.392529:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205236.392532:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.392535:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205236.456423:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205236.456432:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205236.456434:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.456449:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205236.456453:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205236.456456:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205236.456457:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205236.456460:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205236.456462:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205236.456465:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205236.456468:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.456469:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.456474:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205236.456477:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.456480:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205236.456508:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205236.456513:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205236.456515:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205236.456517:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205236.456520:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205236.456525:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6691 00000100:00000001:3.0:1788205236.456529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205236.456531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205236.456532:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:3.0:1788205236.456535:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205236.456539:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205236.456542:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205236.456545:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456547:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205236.456549:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456551:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205236.456554:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205236.456555:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205236.456557:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205236.456560:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205236.456562:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205236.456563:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205236.456565:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456566:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205236.456567:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456568:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205236.456570:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205236.456572:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456573:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205236.456575:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205236.456577:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205236.456579:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456581:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205236.456583:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205236.456590:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205236.456592:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205236.456595:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205236.456596:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456598:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205236.456600:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205236.456603:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205236.456608:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788205236.456614:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205236.456699:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205236.456706:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205236.456709:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205236.456713:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205236.456730:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205236.456733:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205236.456735:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205236.456736:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205236.456737:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205236.456740:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205236.456741:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6691 00000400:00000200:2.0:1788205236.456744:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205236.456746:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205236.456747:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205236.456749:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456751:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205236.456752:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456753:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205236.456755:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205236.456756:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205236.456757:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205236.456758:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205236.456759:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205236.456760:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205236.456762:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456763:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205236.456765:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456766:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205236.456768:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205236.456770:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456771:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205236.456773:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205236.456776:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205236.456778:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456779:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205236.456781:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1788205236.456783:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205236.456786:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1788205236.456787:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456790:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.456791:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1788205236.456793:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205236.456796:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205236.456800:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788205236.456801:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205236.456802:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456805:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205236.456807:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205236.456809:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205236.456814:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:1.0:1788205236.456819:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205236.456820:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.456823:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205236.456827:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205236.456828:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205236.456832:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205236.456835:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205236.456837:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456837:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205236.456845:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205236.456847:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205942400:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205236.456857:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205236.456858:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205236.456862:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788205236.456863:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788205236.456865:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788205236.456867:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456868:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788205236.456872:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788205236.456872:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205236.456877:0:12771:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1788205236.456880:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bec00. 00000100:00080000:2.0:1788205236.456880:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6691 02000000:00000001:1.0:1788205236.456884:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205236.456889:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.456893:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205236.456894:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000010:1.0:1788205236.456899:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000100:00080000:2.0:1788205236.456899:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205236.456904:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456907:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205236.456908:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000040:2.0:1788205236.456910:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456914:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000200:1.0:1788205236.456916:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942400, portal 25 00000020:00000001:2.0:1788205236.456916:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205236.456919:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205236.456921:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788205236.456922:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1788205236.456923:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:1.0:1788205236.456926:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000020:00000001:2.0:1788205236.456927:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205236.456932:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205236.456935:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456938:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205236.456939:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205236.456940:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.456944:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205236.456948:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205236.456954:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205236.456959:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205236.456959:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205236.456962:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205236.456966:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:2.0:1788205236.456966:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205236.456969:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205236.456970:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000001:2.0:1788205236.456971:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205236.456975:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000200:1.0:1788205236.456976:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205942400, offset 0 00000008:00000001:2.0:1788205236.456977:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456980:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205236.456983:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000400:00000200:1.0:1788205236.456987:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000020:00000040:2.0:1788205236.456987:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205236.456992:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205236.456993:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.456994:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205236.456995:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205236.456997:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457001:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788205236.457003:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000001:2.0:1788205236.457007:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457010:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00100000:2.0:1788205236.457013:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205236.457018:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205236.457025:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000001:2.0:1788205236.457027:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205236.457030:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457031:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1788205236.457035:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205236.457038:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457045:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457047:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59218:59218:256:4294967295] 192.168.201.151@tcp LPNI seq info [59198:59198:8:4294967295] 00000400:00000200:1.0:1788205236.457054:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788205236.457061:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205236.457065:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00080000:2.0:1788205236.457067:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205236.457071:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205236.457073:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1788205236.457076:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457078:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205236.457083:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6691 00000800:00000010:1.0:1788205236.457084:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000400:00000200:2.0:1788205236.457091:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000200:1.0:1788205236.457095:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000400:00000200:2.0:1788205236.457100:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000800:00000200:1.0:1788205236.457101:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00080000:2.0:1788205236.457103:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205236.457110:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:1.1:1788205236.457112:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205236.457114:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1788205236.457117:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1788205236.457119:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457123:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205236.457125:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205236.457127:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000100:1.0:1788205236.457128:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000020:00000001:2.0:1788205236.457129:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205236.457132:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000010:1.0:1788205236.457134:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000020:00000001:2.0:1788205236.457134:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205236.457139:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205236.457142:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457144:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205236.457145:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457146:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205236.457148:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205236.457150:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205236.457151:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205236.457152:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205236.457154:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205236.457155:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.457156:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205236.457157:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:2.0:1788205236.457157:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205236.457162:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205236.457164:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457168:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1788205236.457169:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205236.457173:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000400:00000200:1.0:1788205236.457178:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 02000000:00000001:2.0:1788205236.457178:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205236.457180:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205236.457182:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:2.0:1788205236.457184:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205236.457187:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205236.457189:0:2345:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788205236.457190:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205236.457195:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205236.457196:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205236.457202:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205236.457211:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205236.457213:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.457220:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205236.457222:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205236.457225:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205236.457233:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457236:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457240:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205236.457243:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457247:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205236.457247:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6701) 00000400:00000200:1.0:1788205236.457255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205236.457258:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301367809 00000100:00000010:1.0:1788205236.457263:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c15a0. 00000100:00000001:1.0:1788205236.457269:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205236.457275:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301367809 00000100:00000010:1.0:1788205236.457279:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1f00. 00000100:00000001:1.0:1788205236.457281:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205236.457284:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205236.457287:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457289:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205236.457294:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205236/real 1788205236] req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205236.457302:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205236.457303:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205236.457305:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205236.457307:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205236.457309:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205236.457311:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205236.457314:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205236.457322:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457329:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205236.457330:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457332:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.457334:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205236.457338:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457345:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205236.457349:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457357:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205236.457359:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457361:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.457378:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457381:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205236.457383:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205236.457385:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205236.457389:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205236.457391:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205236.457396:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205236.457403:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205942400:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205236.457407:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457409:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.457412:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205942400/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457418:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205236.457421:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205236.457423:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bec00. 02000000:00000001:1.0:1788205236.457426:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205236.457428:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205236.457429:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205236.457431:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205236.457434:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:1.0:1788205236.457437:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205236.457450:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205236.457452:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:1.0:1788205236.457455:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205236.457456:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457458:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457465:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205236.457466:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457468:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205236.457469:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205236.457471:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205236.457474:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457481:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205236.457482:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.457484:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205236.457486:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205236.457488:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205236.457491:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942528:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205236.457494:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205236.457496:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205236.457497:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.457499:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205236.457502:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bec00. 02000000:00000001:1.0:1788205236.457504:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457508:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205236.457510:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205236.457514:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205236.457516:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942528, portal 10 00000100:00000001:1.0:1788205236.457518:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205236.457519:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:1.0:1788205236.457525:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457531:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205236.457534:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205236.457536:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205236.457539:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205942528, offset 0 00000400:00000200:1.0:1788205236.457543:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205236.457547:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.457553:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205236.457557:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205236.457560:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205236.457563:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205236.457566:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205236.457570:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59219:59219:256:4294967295] 192.168.201.151@tcp LPNI seq info [59199:59199:8:4294967295] 00000400:00000200:1.0:1788205236.457574:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.457581:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205236.457586:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.457591:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205236.457595:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205236.457598:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205236.457603:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205236.457605:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205236.457608:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.457610:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205236.457613:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205236.457617:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205236.457620:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205236.457622:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205236.457624:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205236.457628:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457636:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.457638:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457644:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457646:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205236.457648:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205236.457652:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205236.457655:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205236.457657:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301363867 00000100:00000001:1.0:1788205236.457659:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.457661:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301363867 00000100:00000001:1.0:1788205236.457663:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.457664:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205236.457667:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457669:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205236.457672:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205236/real 1788205236] req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205236.457681:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205236.457682:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205236.457684:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205236.457685:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205236.457687:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205236.457689:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205236.457692:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205236.457700:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457707:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205236.457708:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457710:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.457711:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205236.457714:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457721:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205236.457724:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457731:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205236.457733:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457734:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.457735:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457737:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205236.457739:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205236.457741:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205236.457744:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205236.457747:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205236.457750:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205236.457757:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942528:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205236.457761:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205236.457764:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457770:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205236.457772:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.457774:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205236.457775:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205236.457778:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205236.457780:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942656:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205236.457783:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205236.457785:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205236.457787:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.457788:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205236.457792:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bfc00. 02000000:00000001:1.0:1788205236.457793:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.457796:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205236.457798:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205236.457801:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205236.457803:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942656, portal 4 00000100:00000001:1.0:1788205236.457805:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205236.457806:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:1.0:1788205236.457812:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457817:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205236.457819:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205236.457821:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205236.457824:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205942656, offset 0 00000400:00000200:1.0:1788205236.457827:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205236.457830:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.457834:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205236.457837:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205236.457839:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205236.457841:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205236.457843:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205236.457846:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59220:59220:256:4294967295] 192.168.201.151@tcp LPNI seq info [59200:59200:8:4294967295] 00000400:00000200:1.0:1788205236.457850:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.457856:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205236.457860:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.457864:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205236.457867:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205236.457870:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205236.457874:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205236.457876:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205236.457880:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.457881:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205236.457884:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205236.457887:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205236.457890:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205236.457892:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205236.457894:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205236.457897:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457904:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.457908:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457914:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457916:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205236.457919:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205236.457922:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205236.457925:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205236.457927:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301363753 00000100:00000001:1.0:1788205236.457930:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.457932:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301363753 00000100:00000001:1.0:1788205236.457933:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.457935:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205236.457937:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.457939:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205236.457942:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205236/real 1788205236] req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205236.457950:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205236.457951:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205236.457952:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205236.457954:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205236.457955:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205236.457957:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205236.457961:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205236.457969:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457975:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205236.457977:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.457979:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.457980:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205236.457983:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.457991:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205236.457994:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458001:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205236.458003:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458005:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.458006:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458008:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205236.458009:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205236.458012:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205236.458015:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205236.458017:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205236.458021:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205236.458028:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942656:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205236.458031:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205236.458034:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458040:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205236.458042:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.458043:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205236.458045:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205236.458047:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205236.458050:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205236.458053:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205236.458054:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205236.458056:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205236.458057:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205236.458060:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc000. 02000000:00000001:1.0:1788205236.458062:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205236.458066:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205236.458068:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205236.458071:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205236.458073:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942784, portal 4 00000100:00000001:1.0:1788205236.458075:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205236.458076:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:1.0:1788205236.458081:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458087:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205236.458089:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205236.458091:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205236.458093:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205942784, offset 0 00000400:00000200:1.0:1788205236.458096:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205236.458099:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.458102:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205236.458105:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205236.458107:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205236.458110:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205236.458112:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205236.458114:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59221:59221:256:4294967295] 192.168.201.151@tcp LPNI seq info [59201:59201:8:4294967295] 00000400:00000200:1.0:1788205236.458118:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205236.458124:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205236.458128:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.458131:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205236.458135:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205236.458137:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205236.458141:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205236.458144:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205236.458147:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205236.458149:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205236.458152:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205236.458155:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205236.458157:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205236.458159:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205236.458162:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205236.458165:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458172:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.458175:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458181:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.458182:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205236.458185:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205236.458187:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205236.458190:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205236.458192:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301363759 00000100:00000001:1.0:1788205236.458194:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.458196:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301363759 00000100:00000001:1.0:1788205236.458198:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301367809 00000100:00000001:1.0:1788205236.458200:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205236.458201:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.458203:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205236.458205:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205236/real 1788205236] req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205236.458212:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205236.458214:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205236.458215:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205236.458217:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205236.458219:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205236.458221:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205236.458224:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205236.458233:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458244:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205236.458245:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458247:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.458248:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205236.458252:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458259:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205236.458263:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458271:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205236.458273:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458275:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205236.458276:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458278:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205236.458280:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205236.458283:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205236.458286:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205236.458288:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205236.458292:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205236.458299:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205942784:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205236.458302:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458304:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.458307:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205942528/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458314:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205236.458315:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205236.458317:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bec00. 02000000:00000001:1.0:1788205236.458320:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205236.458321:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205236.458322:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205236.458324:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205236.458325:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:1.0:1788205236.458327:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205236.458329:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205236.458330:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:1.0:1788205236.458332:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205236.458334:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458335:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.458338:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205942656/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458345:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205236.458346:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205236.458347:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:1.0:1788205236.458349:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205236.458350:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205236.458351:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205236.458352:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205236.458353:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:1.0:1788205236.458355:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205236.458356:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205236.458357:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:1.0:1788205236.458359:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205236.458360:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458362:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205236.458383:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205942784/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205236.458389:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205236.458390:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205236.458392:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc000. 02000000:00000001:1.0:1788205236.458394:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205236.458396:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205236.458397:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205236.458399:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205236.458401:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:1.0:1788205236.458403:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205236.458404:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205236.458406:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:1.0:1788205236.458408:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205236.458409:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458411:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205236.458415:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205236.458416:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205236.458418:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205236.458420:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205236.648495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205236.648501:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205236.648507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205236.648508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205236.648511:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205236.648513:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205236.648516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205236.648517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205236.648679:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205236.648682:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205236.648685:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205236.968653:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205236.968660:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205236.968662:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205236.968666:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205236.968669:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205236.968674:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.968676:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205236.968678:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.968681:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205237.416516:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205237.416523:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205237.416525:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205237.416530:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205237.416533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205237.416537:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205237.416539:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205237.416541:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205237.416544:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205237.480475:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205237.480475:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205237.480482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205237.480482:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205237.480484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205237.480484:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205237.480489:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:3.0:1788205237.480489:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:1.0:1788205237.480493:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205237.480493:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205237.480497:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.480497:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.480499:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205237.480502:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205237.480505:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205237.480505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205237.480506:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205237.480508:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205237.673163:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205237.673170:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205237.673177:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205237.673179:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205237.673183:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205237.673185:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205237.673189:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205237.673191:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205237.673350:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205237.673354:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205237.673357:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205237.992495:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205237.992502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205237.992504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205237.992509:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205237.992513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205237.992516:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.992520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205237.992522:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.992525:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.441428:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205238.441435:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205238.441437:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.441443:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205238.441446:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205238.441450:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205238.441452:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205238.441455:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205238.441458:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.504503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205238.504511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205238.504514:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.504520:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205238.504524:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205238.504529:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.504532:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.504534:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.504538:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.504626:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205238.504629:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205238.504630:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.504633:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205238.504635:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205238.504637:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205238.504639:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205238.504640:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205238.504642:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205238.696501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205238.696508:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205238.696513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205238.696515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205238.696518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205238.696520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205238.696522:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205238.696524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205238.696587:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205238.696592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205238.696595:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205239.017041:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205239.017048:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205239.017050:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205239.017055:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205239.017059:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205239.017062:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205239.017065:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205239.017068:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205239.017071:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205239.464531:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205239.464537:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205239.464539:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205239.464544:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205239.464549:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205239.464552:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205239.464554:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205239.464556:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205239.464559:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205239.530211:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205239.530217:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205239.530218:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205239.530222:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205239.530225:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205239.530228:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205239.530230:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205239.530231:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205239.530233:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205239.530290:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205239.530292:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205239.530293:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205239.530296:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205239.530298:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205239.530300:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.530302:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205239.530304:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.530305:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205239.720617:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205239.720624:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205239.720630:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205239.720631:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205239.720634:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205239.720635:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205239.720639:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205239.720640:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205239.720792:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205239.720795:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205239.720798:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205240.040476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205240.040485:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205240.040486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205240.040492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205240.040495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205240.040499:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205240.040502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205240.040506:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205240.040509:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205240.488578:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205240.488585:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205240.488587:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205240.488592:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205240.488596:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205240.488600:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205240.488602:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205240.488605:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205240.488618:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205240.553753:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205240.553761:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205240.553763:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205240.553767:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205240.553770:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205240.553774:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205240.553777:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205240.553779:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205240.553782:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205240.555953:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205240.555960:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205240.555962:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205240.555966:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205240.555969:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205240.555972:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205240.555975:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205240.555978:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205240.555980:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205240.744538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205240.744544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205240.744549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205240.744551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205240.744554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205240.744555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205240.744557:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205240.744558:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205240.744751:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205240.744754:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205240.744757:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205241.064517:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301367809, at 4301368320 00000100:00000010:1.0:1788205241.064633:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1f00. 00000100:00000001:3.0:1788205241.064901:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205241.064905:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205241.064907:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205241.064911:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205241.064914:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205241.065006:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.065010:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205241.065012:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.065016:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205241.065230:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301367809, at 4301368320 00000100:00000010:0.0:1788205241.065237:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c15a0. 00000100:00000001:1.0:1788205241.512528:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205241.512536:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205241.512538:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205241.512544:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205241.512547:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205241.512551:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205241.512554:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205241.512556:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205241.512559:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.576528:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205241.576536:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205241.576538:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205241.576543:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205241.576547:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205241.576550:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205241.576553:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205241.576556:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205241.576558:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205241.577052:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205241.577059:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205241.577061:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205241.577064:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205241.577068:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205241.577076:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6696 00000400:00000200:1.0:1788205241.577085:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205241.577099:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205241.577103:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205241.577107:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.577109:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.577112:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.577114:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205241.577118:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205241.577120:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205241.577122:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205241.577125:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205241.577128:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205241.577130:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205241.577133:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.577135:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.577137:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.577138:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205241.577141:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205241.577144:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205241.577147:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205241.577150:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205241.577154:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205241.577157:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.577159:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205241.577161:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205241.577170:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788205241.577173:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205241.577177:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205241.577179:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.577181:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205241.577184:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205241.577187:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205241.577196:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc000. 00000100:00000001:1.0:1788205241.577203:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205241.577213:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205241.577223:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205241.577227:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205241.577231:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.577296:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205241.577300:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205241.577302:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205241.577308:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.577316:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205241.577317:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.577321:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205241.577323:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205241.577325:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205241.577328:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205942912:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205241.577332:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205241.577336:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205241.577338:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.577340:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205241.577346:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcbc00. 02000000:00000001:0.0:1788205241.577348:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205241.577402:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205241.577407:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dac08. 00000400:00000010:0.0:1788205241.577414:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c3192177f8. 00000100:00000200:0.0:1788205241.577426:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205942912, portal 25 00000100:00000001:0.0:1788205241.577429:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205241.577431:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:0.0:1788205241.577437:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.577445:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205241.577448:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205241.577451:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217908. 00000100:00000200:0.0:1788205241.577454:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205942912, offset 0 00000400:00000200:0.0:1788205241.577461:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205241.577467:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.577473:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205241.577479:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205241.577484:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205241.577487:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205241.577490:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205241.577493:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59222:59222:256:4294967295] 192.168.201.151@tcp LPNI seq info [59202:59202:8:4294967295] 00000400:00000200:0.0:1788205241.577498:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.577507:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205241.577513:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205241.577517:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af000. 00000800:00000200:0.0:1788205241.577523:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205241.577528:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205241.577572:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205241.577577:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205241.577579:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301372930 00000100:00000010:0.0:1788205241.577584:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:0.0:1788205241.577590:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205241.577596:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301372930 00000100:00000010:0.0:1788205241.577599:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9d80. 00000100:00000001:0.0:1788205241.577602:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205241.577605:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205241.577608:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.577611:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.577613:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.577664:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205241.577667:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205241.577669:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205241.577672:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205241.577674:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205241.577676:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.577677:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205241.577679:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.577681:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.579000:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205241.579006:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205241.579010:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205241.579012:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205241.579014:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205241.579017:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205241.579021:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6696 00000400:00000200:1.0:1788205241.579028:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205241.579032:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205241.579034:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205241.579037:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.579040:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.579042:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.579044:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205241.579048:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205241.579049:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205241.579051:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205241.579053:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205241.579055:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205241.579058:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788205241.579060:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.579062:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.579064:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.579066:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205241.579069:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205241.579071:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205241.579074:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205241.579076:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205241.579082:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205241.579085:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.579086:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205241.579096:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205241.579099:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.579101:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205241.579107:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788205241.579110:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205241.579113:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205241.579115:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.579117:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205241.579119:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205241.579122:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205241.579127:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788205241.579131:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205241.579138:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205241.579146:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205241.579148:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205241.579153:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.586772:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205241.586778:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205241.586783:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205241.586785:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205241.586788:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205241.586801:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.586812:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205241.586814:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.586818:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205241.586822:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205241.586825:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205241.586828:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943040:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205241.586833:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205241.586836:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205241.586839:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.586841:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205241.586851:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9000. 02000000:00000001:0.0:1788205241.586853:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205241.586860:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205241.586865:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da268. 00000400:00000010:0.0:1788205241.586872:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217a18. 00000100:00000200:0.0:1788205241.586878:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943040, portal 10 00000100:00000001:0.0:1788205241.586880:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205241.586882:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:0.0:1788205241.586889:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.586897:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205241.586899:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205241.586902:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217440. 00000100:00000200:0.0:1788205241.586905:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205943040, offset 0 00000400:00000200:0.0:1788205241.586911:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205241.586920:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.586926:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205241.586932:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205241.586936:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205241.586939:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205241.586942:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205241.586946:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59223:59223:255:4294967295] 192.168.201.151@tcp LPNI seq info [59203:59203:7:4294967295] 00000400:00000200:0.0:1788205241.586950:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.586959:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205241.586966:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205241.586970:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af300. 00000800:00000200:0.0:1788205241.586986:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205241.586992:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205241.586996:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205241.587000:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205241.587002:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205241.587005:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301368989 00000100:00000001:0.0:1788205241.587008:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.587011:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301368989 00000100:00000001:0.0:1788205241.587013:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.587015:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205241.587017:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.587021:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.587023:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205241.587026:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205241.587028:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205241.587029:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205241.587030:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205241.587032:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.587034:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588232:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205241.588240:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205241.588247:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205241.588249:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205241.588252:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205241.588258:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205241.588270:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6696 00000400:00000200:1.0:1788205241.588280:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205241.588284:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205241.588287:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205241.588293:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588296:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.588298:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588303:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205241.588307:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205241.588309:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205241.588311:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205241.588314:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205241.588317:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205241.588322:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205241.588325:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588327:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.588329:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588330:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205241.588333:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205241.588336:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588341:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205241.588343:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205241.588393:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205241.588396:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588398:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205241.588405:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205241.588407:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588409:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205241.588419:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788205241.588423:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205241.588426:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205241.588428:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588433:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205241.588436:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205241.588439:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205241.588456:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788205241.588464:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205241.588475:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205241.588487:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205241.588491:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205241.588498:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205241.588506:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205241.588508:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205241.588515:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205241.588516:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205241.588518:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205241.588521:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205241.588523:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6696 00000400:00000200:1.0:1788205241.588530:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205241.588533:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205241.588535:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205241.588537:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588539:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.588541:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588543:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205241.588610:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205241.588615:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205241.588617:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205241.588620:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205241.588622:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205241.588624:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205241.588630:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588632:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205241.588635:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588637:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205241.588639:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205241.588645:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205241.588648:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205241.588649:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205241.588652:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205241.588654:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588656:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205241.588662:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205241.588665:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588666:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205241.588670:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788205241.588675:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205241.588678:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205241.588679:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205241.588681:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205241.588682:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205241.588683:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205241.588687:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788205241.588694:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205241.588698:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205241.588704:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205241.588709:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205241.588712:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205241.588718:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205241.588726:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6706) 00000100:00000001:0.0:1788205241.589395:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205241.589400:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205241.589402:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205241.589412:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.589419:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205241.589421:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.589423:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205241.589426:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205241.589431:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205241.589434:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943168:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205241.589438:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205241.589454:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205241.589456:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.589458:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205241.589464:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205241.589465:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205241.589470:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205241.589474:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202dae70. 00000400:00000010:0.0:1788205241.589483:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217088. 00000100:00000200:0.0:1788205241.589487:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943168, portal 4 00000100:00000001:0.0:1788205241.589489:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205241.589491:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:0.0:1788205241.589536:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.589546:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205241.589548:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205241.589552:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217550. 00000100:00000200:0.0:1788205241.589555:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205943168, offset 0 00000400:00000200:0.0:1788205241.589563:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205241.589570:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.589579:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205241.589584:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205241.589588:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205241.589591:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205241.589596:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205241.589599:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59224:59224:254:4294967295] 192.168.201.151@tcp LPNI seq info [59204:59204:6:4294967295] 00000400:00000200:0.0:1788205241.589604:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.589615:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205241.589620:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205241.589651:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054afb00. 00000800:00000200:0.0:1788205241.589872:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205241.590048:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205241.590056:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205241.590060:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205241.590064:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205241.590067:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301368885 00000100:00000001:0.0:1788205241.590070:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.590072:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301368885 00000100:00000001:0.0:1788205241.590074:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.590076:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205241.590079:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.590081:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205241.590086:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.590093:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205241.590095:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.590097:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205241.590099:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205241.590101:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205241.590104:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205241.590108:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205241.590110:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205241.590111:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205241.590113:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205241.590117:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8800. 02000000:00000001:0.0:1788205241.590118:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205241.590121:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205241.590124:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da3c8. 00000400:00000010:0.0:1788205241.590533:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217dd0. 00000100:00000200:0.0:1788205241.590539:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943296, portal 4 00000100:00000001:0.0:1788205241.590541:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205241.590543:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00000040:0.0:1788205241.590550:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205241.590557:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205241.590559:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205241.590562:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217b28. 00000100:00000200:0.0:1788205241.590565:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205943296, offset 0 00000400:00000200:0.0:1788205241.590570:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205241.590575:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.590580:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205241.590583:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205241.590586:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205241.590588:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205241.590590:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205241.590593:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59225:59225:253:4294967295] 192.168.201.151@tcp LPNI seq info [59205:59205:5:4294967295] 00000400:00000200:0.0:1788205241.590597:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205241.591050:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205241.591056:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205241.591062:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af900. 00000800:00000200:0.0:1788205241.591067:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205241.591071:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:0.0:1788205241.591074:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205241.591077:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205241.591080:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205241.591081:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301368892 00000100:00000001:0.0:1788205241.591084:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.591086:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301368892 00000100:00000001:0.0:1788205241.591088:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301372930 00000100:00000001:0.0:1788205241.591090:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205241.591092:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.591096:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.591099:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205241.591106:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205241.591109:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788205241.591111:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205241.591112:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205241.591114:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205241.591116:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768700:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205241.768706:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768721:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205241.768725:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205241.768727:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205241.768731:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768733:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205241.768735:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768737:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768739:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768743:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205241.768744:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205241.768747:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205241.768749:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205241.768914:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205241.768918:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205241.768920:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205242.088704:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205242.088711:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205242.088713:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205242.088718:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205242.088721:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205242.088724:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.088727:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205242.088729:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.088732:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205242.536528:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205242.536535:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205242.536537:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205242.536542:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205242.536546:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205242.536549:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205242.536552:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205242.536554:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205242.536557:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205242.600919:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205242.600927:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205242.600928:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205242.600933:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205242.600936:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205242.600940:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.600942:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205242.600944:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.600947:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.601992:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205242.601998:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205242.602000:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205242.602004:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205242.602006:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205242.602009:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205242.602011:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205242.602013:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205242.602015:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205242.793455:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205242.793463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205242.793470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205242.793471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205242.793475:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205242.793476:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205242.793480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205242.793481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205242.793518:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205242.793523:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205242.793527:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205243.112475:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205243.112481:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205243.112484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.112489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205243.112492:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205243.112496:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.112499:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.112502:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.112505:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.560703:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205243.560708:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205243.560710:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.560714:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205243.560717:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205243.560720:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205243.560721:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205243.560723:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205243.560725:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.624489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205243.624496:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205243.624498:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.624504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205243.624508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205243.624511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.624514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.624517:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.624519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.626682:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205243.626688:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205243.626690:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.626696:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205243.626699:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205243.626702:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.626704:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.626707:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.626709:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816491:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205243.816499:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205243.816501:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205243.816505:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816509:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205243.816511:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816513:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816516:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205243.816521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205243.816525:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205243.816527:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205243.816528:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816530:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205243.816531:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816532:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816534:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205243.816539:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205243.816540:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205243.816541:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816543:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205243.816544:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816546:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205243.816547:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205243.816847:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205243.816850:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205243.816853:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205244.136481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205244.136489:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205244.136491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205244.136497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205244.136500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205244.136504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205244.136507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205244.136510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205244.136513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205244.586498:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205244.586507:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205244.586509:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205244.586515:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205244.586519:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205244.586524:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.586526:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205244.586529:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.586532:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205244.648876:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205244.648884:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205244.648886:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205244.648892:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205244.648895:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205244.648899:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205244.648901:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205244.648903:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205244.648906:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788205244.649665:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.151:988 00000100:00000001:1.0:1788205244.650091:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205244.650095:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205244.650097:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205244.650101:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205244.650104:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205244.650107:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.650109:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205244.650111:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.650112:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788205244.650569:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.151@tcp at host 192.168.201.151 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788205244.650580:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205244.650583:0:2335:0:(router.c:1791:lnet_notify()) 192.168.201.51@tcp notifying 192.168.201.151@tcp: down 00000400:00000200:2.0:1788205244.650588:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205244.650594:0:2335:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1788205244.650598:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788205244.650601:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af000. 00000400:00000200:2.0:1788205244.650609:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205244.650613:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205244.650619:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217908 00000400:00000010:2.0:1788205244.650621:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217908. 00000100:00000001:2.0:1788205244.650626:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205244.650634:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650659:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205244.650663:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650671:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.650673:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205244.650681:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788205244.650683:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af300. 00000400:00000200:2.0:1788205244.650687:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205244.650690:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205244.650693:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217440 00000400:00000010:2.0:1788205244.650694:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217440. 00000100:00000001:2.0:1788205244.650696:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205244.650875:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650886:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205244.650890:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650895:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.650897:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205244.650903:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788205244.650905:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054afb00. 00000400:00000200:2.0:1788205244.650908:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205244.650912:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205244.650916:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217550 00000400:00000010:2.0:1788205244.650917:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217550. 00000100:00000001:2.0:1788205244.650920:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205244.650923:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650930:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205244.650933:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650938:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.650939:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205244.650944:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.151@tcp 00000800:00000010:2.0:1788205244.650946:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af900. 00000400:00000200:2.0:1788205244.650949:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205244.650951:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205244.650955:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217b28 00000400:00000010:2.0:1788205244.650956:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217b28. 00000100:00000001:2.0:1788205244.650958:0:2335:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205244.650961:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650968:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205244.650970:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205244.650975:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.650976:0:2335:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205244.651939:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205244.651944:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205244.651947:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205244.651957:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205241/real 1788205244] req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205244.651972:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205244.651975:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205244.651978:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dae70. 00000400:00000200:1.0:1788205244.651982:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217088 00000400:00000010:1.0:1788205244.651984:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217088. 00000100:00000001:1.0:1788205244.651986:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205244.651990:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205244.651996:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652002:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205244.652004:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652007:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652009:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205244.652013:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652019:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205244.652024:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652032:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205244.652035:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652037:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652038:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652041:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205244.652045:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205244.652047:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205244.652050:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205244.652052:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205244.652055:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205244.652060:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943168:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205244.652063:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205244.652066:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205241/real 1788205244] req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205244.652071:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205244.652072:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205244.652073:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da3c8. 00000400:00000200:1.0:1788205244.652075:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217dd0 00000400:00000010:1.0:1788205244.652076:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217dd0. 00000100:00000001:1.0:1788205244.652077:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205244.652079:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205244.652084:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652090:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205244.652091:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652093:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652094:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205244.652098:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652108:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205244.652112:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652118:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205244.652120:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652122:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652123:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652125:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205244.652127:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205244.652129:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205244.652132:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205244.652134:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205244.652136:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205244.652141:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943296:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205244.652143:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205244.652145:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205241/real 1788205244] req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205244.652150:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205244.652151:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205244.652153:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da268. 00000400:00000200:1.0:1788205244.652155:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217a18 00000400:00000010:1.0:1788205244.652156:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217a18. 00000100:00000001:1.0:1788205244.652157:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205244.652159:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205244.652167:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652171:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205244.652172:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652173:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652174:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205244.652175:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652179:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205244.652181:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652184:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205244.652185:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652186:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652187:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652188:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205244.652189:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205244.652190:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205244.652192:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205244.652194:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205244.652195:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205244.652199:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943040:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205244.652201:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205244.652204:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205241/real 1788205244] req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205244.652214:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205244.652215:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205244.652217:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202dac08. 00000400:00000200:1.0:1788205244.652219:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c3192177f8 00000400:00000010:1.0:1788205244.652221:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c3192177f8. 00000100:00000001:1.0:1788205244.652223:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205244.652226:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205244.652232:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652238:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205244.652239:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652241:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652242:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205244.652245:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652252:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205244.652255:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652262:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205244.652264:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652265:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205244.652267:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652268:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205244.652270:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205244.652272:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205244.652275:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205244.652277:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205244.652281:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205244.652288:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205942912:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205244.652291:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652293:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205244.652295:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205943168/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652312:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205244.652315:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205244.652319:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:1.0:1788205244.652322:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205244.652328:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205244.652330:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205244.652333:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205244.652335:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:1.0:1788205244.652339:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205244.652340:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205244.652342:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:1.0:1788205244.652345:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205244.652346:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652347:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205244.652351:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205943296/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652356:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205244.652357:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205244.652359:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8800. 02000000:00000001:1.0:1788205244.652361:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205244.652362:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205244.652408:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205244.652410:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205244.652412:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:1.0:1788205244.652415:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205244.652416:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205244.652417:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:1.0:1788205244.652420:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205244.652421:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652422:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205244.652426:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205943040/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652431:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205244.652432:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205244.652434:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9000. 02000000:00000001:1.0:1788205244.652436:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205244.652437:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205244.652438:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205244.652440:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205244.652441:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:1.0:1788205244.652443:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205244.652444:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205244.652445:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:1.0:1788205244.652447:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205244.652448:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652450:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205244.652452:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205942912/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205244.652458:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205244.652459:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205244.652460:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcbc00. 02000000:00000001:1.0:1788205244.652462:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205244.652463:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205244.652464:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205244.652465:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205244.652467:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc000. 02000000:00000001:1.0:1788205244.652471:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205244.652473:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205244.652474:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:1.0:1788205244.652476:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205244.652477:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652479:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205244.652487:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205244.652489:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205244.652491:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205244.652493:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205244.840463:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205244.840469:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205244.840473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205244.840475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205244.840477:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205244.840478:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205244.840480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205244.840481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205244.840562:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205244.840567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205244.840570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205245.160484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205245.160491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205245.160494:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205245.160499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205245.160502:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205245.160505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205245.160508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205245.160511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205245.160517:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205245.608624:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205245.608633:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205245.608635:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205245.608641:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205245.608644:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205245.608649:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205245.608651:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205245.608653:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205245.608656:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205245.672670:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205245.672678:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205245.672680:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205245.672685:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205245.672689:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205245.672693:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205245.672696:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205245.672698:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205245.672701:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205245.673441:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205245.673446:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205245.673448:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205245.673452:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205245.673454:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205245.673457:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205245.673458:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205245.673470:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205245.673473:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205245.864493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205245.864499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205245.864502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205245.864503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205245.864505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205245.864506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205245.864508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205245.864509:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205245.864629:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205245.864632:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205245.864635:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205246.184478:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205246.184485:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205246.184488:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205246.184493:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205246.184496:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205246.184500:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.184502:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205246.184505:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.184508:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205246.185133:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301372930, at 4301373440 00000100:00000010:0.0:1788205246.185143:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000040:1.0:1788205246.185791:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301372930, at 4301373441 00000100:00000010:1.0:1788205246.185798:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9d80. 00000100:00000001:1.0:1788205246.632456:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205246.632463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205246.632465:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205246.632470:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205246.632473:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205246.632477:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205246.632480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205246.632482:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205246.632485:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205246.696525:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205246.696533:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205246.696535:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205246.696540:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205246.696544:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205246.696548:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.696551:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205246.696554:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.696557:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205246.696686:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205246.696692:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205246.696695:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205246.696698:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205246.696702:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205246.696708:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6701 00000400:00000200:1.0:1788205246.696717:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205246.696723:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205246.696726:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205246.696729:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696732:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.696735:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696737:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205246.696740:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205246.696742:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205246.696745:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205246.696748:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205246.696750:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205246.696752:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205246.696755:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696757:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.696759:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696761:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205246.696764:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205246.696767:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696770:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205246.696772:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205246.696775:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205246.696778:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.696781:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205246.696783:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205246.696792:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536a00. 00000020:00000040:1.0:1788205246.696796:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205246.696800:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205246.696802:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.696805:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205246.696807:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205246.696812:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205246.696820:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bfc00. 00000100:00000001:1.0:1788205246.696827:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205246.696837:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205246.696848:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205246.696851:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205246.696856:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9fbd040] to pc [ptlrpcd_rcv+-1] req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205246.696884:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205246.696888:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205246.696891:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205246.696892:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205246.696893:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205246.696896:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205246.696899:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6701 00000400:00000200:1.0:1788205246.696903:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205246.696905:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205246.696907:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205246.696910:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696912:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.696914:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696916:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205246.696918:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205246.696919:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205246.696920:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205246.696922:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:1.0:1788205246.696924:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:1.0:1788205246.696926:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:1.0:1788205246.696929:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696931:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.696932:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696934:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205246.696936:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:1.0:1788205246.696938:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205246.696940:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205246.696942:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205246.696947:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205246.696949:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.696951:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205246.696955:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205246.696957:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.696959:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205246.696962:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536300. 00000020:00000040:1.0:1788205246.696964:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205246.696967:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205246.696968:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.696970:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205246.696971:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205246.696973:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205246.696976:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd000. 00000100:00000001:1.0:1788205246.696979:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205246.696984:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205246.696991:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205246.696993:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205246.696997:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093196a45] to pc [ptlrpcd_rcv+-1] req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205246.697004:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205246.697006:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205246.697009:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205246.697010:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205246.697012:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205246.697014:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205246.697016:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6701 00000400:00000200:1.0:1788205246.697020:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205246.697022:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205246.697024:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205246.697027:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697029:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.697030:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697032:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205246.697034:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205246.697035:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205246.697036:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205246.697038:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:1.0:1788205246.697040:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205246.697042:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:1.0:1788205246.697045:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697046:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.697048:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697050:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205246.697052:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205246.697054:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697056:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205246.697057:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205246.697059:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205246.697061:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697063:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205246.697065:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205246.697068:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697069:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205246.697072:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310535880. 00000020:00000040:1.0:1788205246.697074:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:1.0:1788205246.697076:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205246.697078:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697079:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205246.697080:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205246.697082:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205246.697085:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bec00. 00000100:00000001:1.0:1788205246.697087:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205246.697092:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205246.697098:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205246.697100:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205246.697103:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070649401] to pc [ptlrpcd_rcv+-1] req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205246.697110:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205246.697112:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205246.697115:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205246.697117:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205246.697118:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205246.697120:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205246.697122:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6701 00000400:00000200:1.0:1788205246.697125:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205246.697128:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205246.697130:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205246.697132:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697134:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.697136:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697138:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205246.697141:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205246.697142:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:1.0:1788205246.697143:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205246.697145:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:1.0:1788205246.697147:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:1.0:1788205246.697149:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:1.0:1788205246.697152:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697154:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205246.697156:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697157:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205246.697159:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:1.0:1788205246.697161:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205246.697164:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205246.697165:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205246.697168:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205246.697170:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697172:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205246.697175:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205246.697177:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697179:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205246.697183:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310537100. 00000020:00000040:1.0:1788205246.697185:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:1.0:1788205246.697188:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205246.697189:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205246.697191:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205246.697193:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205246.697194:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205246.697197:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc400. 00000100:00000001:1.0:1788205246.697200:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205246.697205:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205246.697211:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205246.697213:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205246.697216:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ef77a8e] to pc [ptlrpcd_rcv+-1] req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205246.697223:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205246.697228:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6711) 00000100:00000001:3.0:1788205246.697298:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205246.697301:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205246.697302:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205246.697305:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205246.697307:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205246.697309:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.697311:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205246.697312:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.697315:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.697394:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205246.697398:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205246.697400:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205246.697407:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697414:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205246.697416:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.697419:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205246.697422:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205246.697424:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205246.697427:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205943424:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205246.697432:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205246.697435:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205246.697438:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.697440:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205246.697445:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205246.697447:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205246.697453:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205246.697457:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205246.697463:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205246.697470:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943424, portal 25 00000100:00000001:0.0:1788205246.697472:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205246.697474:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292787712 : -113511416763904 : ffff98c310536a00) 00000100:00000040:0.0:1788205246.697480:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697487:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205246.697489:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205246.697492:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205246.697495:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205943424, offset 0 00000400:00000200:0.0:1788205246.697500:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205246.697508:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.697514:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205246.697519:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205246.697524:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205246.697528:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205246.697530:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205246.697534:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59226:59226:256:4294967295] 192.168.201.151@tcp LPNI seq info [59206:59206:8:4294967295] 00000400:00000200:0.0:1788205246.697538:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.697547:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205246.697553:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.697558:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205246.697565:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205246.697569:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205246.697574:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205246.697577:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205246.697581:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.697584:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205246.697591:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205246.697595:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205246.697600:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205246.697603:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205246.697606:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205246.697610:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697620:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.697624:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697631:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.697632:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205246.697638:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205246.697641:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205246.697644:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301378050 00000100:00000010:0.0:1788205246.697649:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9420. 00000100:00000001:0.0:1788205246.697655:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205246.697661:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301378050 00000100:00000010:0.0:1788205246.697664:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9d80. 00000100:00000001:0.0:1788205246.697668:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788205246.697672:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205246.697674:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.697676:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205246.697681:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205246/real 1788205246] req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205246.697689:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205246.697691:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205246.697693:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205246.697696:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205246.697697:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205246.697700:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205246.697703:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205246.697712:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697718:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205246.697720:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.697722:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.697724:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205246.697728:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697734:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205246.697739:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697745:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205246.697748:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.697749:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.697750:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.697753:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205246.697755:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205246.697758:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205246.697761:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205246.697763:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205246.697767:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205246.697774:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205943424:192.168.201.151@tcp:250:kworker.0 00000100:00000001:0.0:1788205246.697778:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205246.697781:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697787:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205246.697788:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.697790:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205246.697792:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205246.697795:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205246.697797:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943552:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205246.697800:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205246.697802:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205246.697803:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.697804:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205246.697808:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788205246.697810:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205246.697813:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205246.697815:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205246.697818:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205246.697820:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943552, portal 10 00000100:00000001:0.0:1788205246.697822:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205246.697823:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292785920 : -113511416765696 : ffff98c310536300) 00000100:00000040:0.0:1788205246.697830:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697836:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205246.697838:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205246.697841:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205246.697843:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205943552, offset 0 00000400:00000200:0.0:1788205246.697847:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205246.697851:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.697854:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205246.697857:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205246.697859:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205246.697862:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205246.697864:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205246.697867:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59227:59227:256:4294967295] 192.168.201.151@tcp LPNI seq info [59207:59207:8:4294967295] 00000400:00000200:0.0:1788205246.697871:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.697877:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205246.697881:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.697885:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205246.697888:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205246.697891:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205246.697895:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205246.697897:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205246.697901:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.697903:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205246.697906:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205246.697909:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205246.697912:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205246.697913:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205246.697915:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205246.697919:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697926:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.697929:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.697935:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.697936:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205246.697939:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205246.697943:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205246.697946:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205246.697947:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301374107 00000100:00000001:0.0:1788205246.697950:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.697952:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301374107 00000100:00000001:0.0:1788205246.697954:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.697956:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205246.697958:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.697960:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205246.697963:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205246/real 1788205246] req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205246.697977:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205246.697978:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205246.697980:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205246.697982:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205246.697983:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205246.697986:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205246.697989:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205246.697997:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698003:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205246.698005:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698006:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698007:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205246.698011:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698017:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205246.698021:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698027:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205246.698029:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698031:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698032:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698033:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205246.698036:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205246.698038:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205246.698042:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205246.698044:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205246.698048:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205246.698056:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943552:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205246.698060:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205246.698063:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698068:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205246.698070:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.698071:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205246.698073:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205246.698075:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205246.698077:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943680:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205246.698080:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205246.698082:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205246.698083:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.698085:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698088:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788205246.698090:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205246.698093:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205246.698095:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205246.698098:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205246.698101:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943680, portal 4 00000100:00000001:0.0:1788205246.698102:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205246.698104:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292783232 : -113511416768384 : ffff98c310535880) 00000100:00000040:0.0:1788205246.698109:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698115:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205246.698117:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205246.698120:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205246.698122:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205943680, offset 0 00000400:00000200:0.0:1788205246.698126:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205246.698130:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.698133:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205246.698136:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205246.698138:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205246.698140:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205246.698143:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205246.698146:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59228:59228:256:4294967295] 192.168.201.151@tcp LPNI seq info [59208:59208:8:4294967295] 00000400:00000200:0.0:1788205246.698150:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.698155:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205246.698159:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.698163:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205246.698166:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205246.698169:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205246.698173:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205246.698175:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205246.698179:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.698181:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205246.698183:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205246.698187:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205246.698190:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205246.698191:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205246.698193:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205246.698197:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698204:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698207:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698213:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.698214:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205246.698217:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205246.698219:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205246.698222:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205246.698224:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301373993 00000100:00000001:0.0:1788205246.698226:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.698228:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301373993 00000100:00000001:0.0:1788205246.698230:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.698232:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205246.698233:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.698235:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205246.698239:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205246/real 1788205246] req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205246.698246:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205246.698247:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205246.698248:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205246.698250:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205246.698252:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205246.698254:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205246.698257:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205246.698280:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698287:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205246.698288:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698290:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698291:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205246.698294:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698301:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205246.698304:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698311:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205246.698313:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698315:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698316:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698318:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205246.698320:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205246.698322:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205246.698325:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205246.698327:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205246.698330:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205246.698337:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310535880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943680:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205246.698341:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205246.698344:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698351:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205246.698352:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.698354:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205246.698356:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205246.698358:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205246.698360:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205246.698386:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205246.698388:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205246.698390:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205246.698391:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698395:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205246.698397:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205246.698400:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205246.698403:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205246.698407:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205246.698410:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943808, portal 4 00000100:00000001:0.0:1788205246.698412:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205246.698414:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292789504 : -113511416762112 : ffff98c310537100) 00000100:00000040:0.0:1788205246.698419:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698425:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205246.698428:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205246.698430:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205246.698432:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205943808, offset 0 00000400:00000200:0.0:1788205246.698436:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205246.698439:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.698442:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205246.698445:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205246.698447:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205246.698449:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205246.698452:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205246.698455:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59229:59229:256:4294967295] 192.168.201.151@tcp LPNI seq info [59209:59209:8:4294967295] 00000400:00000200:0.0:1788205246.698458:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205246.698464:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205246.698468:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.698471:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205246.698474:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205246.698477:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788205246.698481:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205246.698483:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788205246.698487:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205246.698489:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205246.698492:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205246.698495:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205246.698498:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205246.698500:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205246.698502:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205246.698505:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698518:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698521:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205271 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698527:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.698528:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205246.698530:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205246.698533:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205246.698536:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205246.698537:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301374000 00000100:00000001:0.0:1788205246.698540:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.698542:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301374000 00000100:00000001:0.0:1788205246.698543:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301378050 00000100:00000001:0.0:1788205246.698545:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205246.698547:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.698548:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205246.698552:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205246/real 1788205246] req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205246.698558:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205246.698559:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205246.698560:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205246.698563:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205246.698564:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205246.698566:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205246.698569:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205246.698576:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698581:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205246.698583:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698584:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698586:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205246.698589:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698594:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205246.698597:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698603:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205246.698605:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698606:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205246.698607:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698609:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205246.698610:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205246.698613:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205246.698617:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205246.698619:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205246.698622:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205246.698628:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310537100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205943808:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205246.698631:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698634:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698636:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536a00 x1875062205943424/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698641:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205246.698644:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698646:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca800. 02000000:00000001:0.0:1788205246.698649:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205246.698651:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205246.698654:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:0.0:1788205246.698656:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205246.698658:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bfc00. 02000000:00000001:0.0:1788205246.698661:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205246.698662:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205246.698664:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536a00. 00000100:00000001:0.0:1788205246.698668:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205246.698669:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698670:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698672:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536300 x1875062205943552/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698677:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205246.698679:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698680:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:0.0:1788205246.698683:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205246.698684:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205246.698685:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205246.698686:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205246.698687:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd000. 02000000:00000001:0.0:1788205246.698689:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205246.698690:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205246.698691:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536300. 00000100:00000001:0.0:1788205246.698693:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205246.698694:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698695:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698697:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310535880 x1875062205943680/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698703:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205246.698704:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698705:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dc9400. 02000000:00000001:0.0:1788205246.698707:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205246.698708:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205246.698709:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205246.698710:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205246.698712:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bec00. 02000000:00000001:0.0:1788205246.698714:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205246.698715:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205246.698716:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310535880. 00000100:00000001:0.0:1788205246.698718:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205246.698719:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698721:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205246.698724:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310537100 x1875062205943808/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205271 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205246.698729:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205246.698730:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205246.698732:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca400. 02000000:00000001:0.0:1788205246.698734:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205246.698735:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205246.698736:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205246.698738:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205246.698739:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc400. 02000000:00000001:0.0:1788205246.698741:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205246.698742:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205246.698743:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310537100. 00000100:00000001:0.0:1788205246.698745:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205246.698746:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698748:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205246.698755:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205246.698757:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205246.698760:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205246.698761:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205246.888484:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205246.888490:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205246.888495:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205246.888497:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205246.888500:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205246.888502:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205246.888505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205246.888506:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205246.889102:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205246.889106:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205246.889109:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205247.209242:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205247.209248:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205247.209250:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205247.209355:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205247.209359:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205247.210214:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.210219:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205247.210223:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.210226:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205247.656576:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205247.656584:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205247.656587:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205247.656592:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205247.656595:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205247.656599:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205247.656603:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205247.656605:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205247.656608:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205247.720666:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205247.720672:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205247.720673:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205247.720677:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205247.720679:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205247.720682:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.720684:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205247.720686:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.720688:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205247.720741:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205247.720743:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205247.720744:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205247.720746:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205247.720747:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205247.720748:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205247.720749:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205247.720751:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205247.720752:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205247.912537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205247.912544:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205247.912550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205247.912551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205247.912554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205247.912555:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205247.912558:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205247.912559:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205247.912629:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205247.912633:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205247.912636:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205248.232480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205248.232488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205248.232490:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205248.232496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205248.232499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205248.232504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.232507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205248.232510:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.232513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.680453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205248.680458:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205248.680459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.680463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205248.680471:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205248.680474:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.680477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205248.680479:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.680482:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205248.744474:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205248.744474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205248.744481:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205248.744481:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205248.744483:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.744483:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205248.744488:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205248.744488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:2.0:1788205248.744491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205248.744491:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205248.744495:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.744495:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205248.744498:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205248.744500:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205248.744503:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.744504:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205248.744505:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.744506:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205248.936498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205248.936505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205248.936514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205248.936516:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205248.936520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205248.936521:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205248.936524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205248.936525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205248.936587:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205248.936592:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205248.936595:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205249.256492:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205249.256502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205249.256504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205249.256510:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205249.256514:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205249.256518:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.256520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205249.256523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.256532:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205249.704497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205249.704508:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205249.704511:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205249.704519:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205249.704523:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205249.704528:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205249.704531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205249.704534:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205249.704538:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205249.768466:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205249.768474:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205249.768476:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205249.768480:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205249.768481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205249.768483:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205249.768487:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.768490:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205249.768491:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205249.768495:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205249.768496:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205249.768505:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205249.768507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205249.768512:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205249.768516:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.768520:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205249.768523:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.768526:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205249.960709:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205249.960719:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205249.960726:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205249.960728:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205249.960732:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205249.960733:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205249.960737:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205249.960739:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205249.960798:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205249.960802:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205249.960804:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205250.281310:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205250.281414:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205250.281418:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205250.281425:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205250.281429:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205250.281435:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205250.281438:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205250.281441:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205250.281445:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205250.728509:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205250.728520:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205250.728522:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205250.728529:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205250.728533:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205250.728538:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205250.728541:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205250.728544:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205250.728548:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205250.792510:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205250.792519:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205250.792520:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205250.792526:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205250.792529:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205250.792533:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205250.792536:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205250.792538:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205250.792541:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205250.792654:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205250.792658:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205250.792659:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205250.792662:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205250.792663:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205250.792665:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205250.792667:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205250.792668:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205250.792669:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205250.984509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205250.984517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205250.984525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205250.984527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205250.984531:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205250.984533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205250.984537:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205250.984538:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205250.984594:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205250.984599:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205250.984603:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205251.304496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1788205251.304501:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301378050, at 4301378560 00000100:00000001:3.0:1788205251.304502:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205251.304504:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205251.304508:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205251.304510:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205251.304512:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205251.304516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205251.304519:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205251.304521:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205251.304523:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9d80. 00000100:00000040:0.0:1788205251.305681:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301378050, at 4301378561 00000100:00000010:0.0:1788205251.305694:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9420. 00000100:00000001:1.0:1788205251.752481:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205251.752488:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205251.752490:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205251.752495:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205251.752498:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205251.752502:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.752504:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205251.752507:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.752509:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205251.816460:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205251.816480:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205251.816484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205251.816484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205251.816491:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205251.816494:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205251.816495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205251.816500:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205251.816503:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205251.816505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205251.816509:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788205251.816509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205251.816510:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205251.816517:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205251.816522:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205251.816522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205251.816534:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788205251.816534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205251.816542:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205251.816543:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205251.816548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205251.816549:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205251.816555:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205251.816561:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6706 00000400:00000200:0.0:1788205251.816572:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205251.816579:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205251.816582:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205251.816585:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816588:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816589:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816591:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205251.816595:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205251.816596:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:0.0:1788205251.816599:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205251.816602:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:0.0:1788205251.816604:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:0.0:1788205251.816606:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:0.0:1788205251.816609:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816611:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816613:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816615:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205251.816618:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:0.0:1788205251.816621:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816624:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205251.816627:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205251.816630:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205251.816633:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816635:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205251.816637:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205251.816645:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ec380. 00000020:00000040:0.0:1788205251.816648:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:0.0:1788205251.816651:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205251.816652:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816654:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205251.816656:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205251.816659:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205251.816665:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc9400. 00000100:00000001:0.0:1788205251.816670:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205251.816676:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205251.816683:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205251.816686:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205251.816689:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa67e59] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205251.816713:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205251.816715:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205251.816718:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205251.816719:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205251.816720:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205251.816722:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205251.816723:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6706 00000400:00000200:0.0:1788205251.816725:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205251.816727:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205251.816728:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205251.816731:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816732:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816734:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816735:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205251.816736:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205251.816737:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:0.0:1788205251.816738:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205251.816740:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:0.0:1788205251.816741:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:0.0:1788205251.816743:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:0.0:1788205251.816745:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816746:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816748:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816749:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205251.816751:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:0.0:1788205251.816752:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816754:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205251.816755:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205251.816755:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788205251.816761:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205251.816761:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205251.816765:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788205251.816766:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816770:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205251.816774:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1788205251.816775:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788205251.816779:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816782:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205251.816785:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ef480. 00000020:00000040:0.0:1788205251.816790:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:0.0:1788205251.816796:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205251.816798:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.816800:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205251.816801:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205251.816803:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205251.816804:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205251.816808:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.816811:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:0.0:1788205251.816813:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dc8c00. 02000000:00000001:1.0:1788205251.816815:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205251.816819:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.816820:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205251.816826:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205251.816827:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205943936:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205251.816834:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205251.816839:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205251.816841:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205251.816841:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816846:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.816846:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205251.816852:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 00000100:00000040:0.0:1788205251.816853:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1d16181] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205251.816857:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816866:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.816868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:0.0:1788205251.816869:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205251.816873:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205251.816874:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1788205251.816874:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000100:00080000:0.0:1788205251.816875:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205251.816879:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205251.816881:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6706 00000400:00000200:0.0:1788205251.816883:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205251.816886:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205251.816888:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205251.816890:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000010:1.0:1788205251.816891:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000001:0.0:1788205251.816892:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816895:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816897:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:0.0:1788205251.816899:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205251.816900:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000200:1.0:1788205251.816900:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205943936, portal 25 00000020:00000001:0.0:1788205251.816901:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205251.816904:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:1.0:1788205251.816904:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788205251.816906:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000001:1.0:1788205251.816907:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293531520 : -113511416020096 : ffff98c3105ec380) 00000020:00000001:0.0:1788205251.816909:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:0.0:1788205251.816913:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816915:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.816918:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:1.0:1788205251.816919:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205251.816920:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:0.0:1788205251.816925:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:0.0:1788205251.816930:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205251.816935:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.816937:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788205251.816939:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205251.816943:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000001:0.0:1788205251.816944:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205251.816948:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205251.816948:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000001:0.0:1788205251.816950:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:1.0:1788205251.816952:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205943936, offset 0 00000008:00100000:0.0:1788205251.816954:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:0.0:1788205251.816958:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.816961:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.816962:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000010:0.0:1788205251.816968:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105edc00. 00000020:00000040:0.0:1788205251.816971:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:0.0:1788205251.816973:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205251.816975:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.816975:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205251.816978:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205251.816980:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205251.816982:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.816985:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 02000000:00000010:0.0:1788205251.816986:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca800. 00000100:00000001:0.0:1788205251.816992:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.816995:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1788205251.816998:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205251.817003:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205251.817010:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788205251.817013:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205251.817013:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.817018:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.817019:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59230:59230:256:4294967295] 192.168.201.151@tcp LPNI seq info [59210:59210:8:4294967295] 00000400:00000200:1.0:1788205251.817025:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:0.0:1788205251.817027:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e626635c] to pc [ptlrpcd_rcv+-1] req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205251.817038:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:0.0:1788205251.817044:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205251.817047:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205251.817051:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:1.0:1788205251.817051:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000100:00000001:0.0:1788205251.817053:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205251.817054:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205251.817057:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205251.817059:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6706 00000400:00000200:0.0:1788205251.817064:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205251.817066:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205251.817067:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000800:00000010:1.0:1788205251.817067:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000100:00000040:0.0:1788205251.817072:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.817074:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.817076:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.817081:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:1.0:1788205251.817081:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:0.0:1788205251.817087:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788205251.817088:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000200:1.0:1788205251.817088:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000020:00000001:0.0:1788205251.817092:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788205251.817096:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000800:00000200:1.1:1788205251.817097:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:0.0:1788205251.817099:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000200:1.1:1788205251.817102:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000020:00000001:0.0:1788205251.817104:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000100:1.0:1788205251.817108:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000100:00000040:0.0:1788205251.817112:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817112:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000100:00000001:0.0:1788205251.817117:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205251.817120:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205251.817122:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:1.0:1788205251.817122:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:0.0:1788205251.817128:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:0.0:1788205251.817132:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817134:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788205251.817136:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205251.817140:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205251.817141:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 02000000:00000001:0.0:1788205251.817143:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205251.817147:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.817149:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205251.817151:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:0.0:1788205251.817153:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205251.817153:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:0.0:1788205251.817156:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817160:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205251.817166:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788205251.817168:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c3105ed500. 00000100:00000001:1.0:1788205251.817194:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1788205251.817199:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000040:1.0:1788205251.817199:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205251.817204:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205251.817207:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205251.817213:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205251.817213:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205251.817215:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205251.817216:0:2345:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1788205251.817219:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817223:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205251.817224:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310dca400. 00000100:00000001:0.0:1788205251.817229:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.817229:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205251.817234:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301383169 00000100:00100000:0.0:1788205251.817237:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1788205251.817240:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1f00. 00000100:00000001:0.0:1788205251.817247:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205251.817249:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788205251.817250:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205251.817254:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005796579d] to pc [ptlrpcd_rcv+-1] req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817258:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301383169 00000100:00000001:0.0:1788205251.817259:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205251.817265:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1840. 00000100:00000040:0.0:1788205251.817266:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6716) 00000100:00000001:1.0:1788205251.817272:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205251.817277:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205251.817281:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817282:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205251.817287:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205251/real 1788205251] req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205251.817294:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205251.817296:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205251.817297:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205251.817299:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205251.817300:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205251.817301:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205251.817303:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205251.817310:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817316:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205251.817318:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817320:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817322:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205251.817325:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817331:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205251.817335:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817341:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205251.817343:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817345:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817346:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817348:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205251.817350:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205251.817353:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205251.817356:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205251.817359:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205251.817384:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205251.817392:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205943936:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205251.817396:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817398:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.817401:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ec380 x1875062205943936/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817407:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205251.817409:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205251.817411:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205251.817415:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205251.817416:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205251.817418:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205251.817420:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205251.817421:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc9400. 02000000:00000001:1.0:1788205251.817424:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205251.817425:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205251.817426:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ec380. 00000100:00000001:1.0:1788205251.817429:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205251.817429:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817431:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817436:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205251.817437:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817439:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205251.817440:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205251.817440:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205251.817443:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817447:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205251.817448:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817450:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205251.817451:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205251.817452:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205251.817454:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944064:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205251.817456:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205251.817458:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205251.817459:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817460:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205251.817462:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205251.817463:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.817466:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205251.817467:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205251.817469:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205251.817471:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944064, portal 10 00000100:00000001:1.0:1788205251.817473:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205251.817474:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293544064 : -113511416007552 : ffff98c3105ef480) 00000100:00000040:1.0:1788205251.817478:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817481:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205251.817483:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205251.817484:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205251.817485:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205944064, offset 0 00000400:00000200:1.0:1788205251.817487:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817490:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817492:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205251.817495:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205251.817498:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205251.817500:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205251.817502:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817505:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59231:59231:256:4294967295] 192.168.201.151@tcp LPNI seq info [59211:59211:8:4294967295] 00000400:00000200:1.0:1788205251.817508:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817514:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205251.817519:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817523:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205251.817526:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205251.817529:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205251.817534:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205251.817536:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205251.817540:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817541:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205251.817544:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205251.817547:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205251.817549:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205251.817550:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205251.817552:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205251.817556:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817563:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.817566:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817573:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817574:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205251.817577:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205251.817581:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205251.817584:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205251.817586:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301379227 00000100:00000001:1.0:1788205251.817589:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.817591:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301379227 00000100:00000001:1.0:1788205251.817594:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.817595:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205251.817597:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817599:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205251.817603:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205251/real 1788205251] req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205251.817617:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205251.817618:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205251.817620:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205251.817623:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205251.817624:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205251.817626:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205251.817629:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205251.817634:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817638:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205251.817639:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817640:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817641:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205251.817643:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817647:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205251.817649:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817653:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205251.817654:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817655:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817656:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817657:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205251.817658:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205251.817659:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205251.817661:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205251.817662:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205251.817665:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205251.817676:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944064:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205251.817678:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205251.817680:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817684:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205251.817685:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817686:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205251.817688:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205251.817689:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205251.817690:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205251.817692:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205251.817693:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205251.817694:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817696:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205251.817699:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205251.817701:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.817704:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205251.817706:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205251.817709:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205251.817711:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944192, portal 4 00000100:00000001:1.0:1788205251.817713:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205251.817714:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293537792 : -113511416013824 : ffff98c3105edc00) 00000100:00000040:1.0:1788205251.817720:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817726:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205251.817728:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205251.817731:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205251.817733:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205944192, offset 0 00000400:00000200:1.0:1788205251.817737:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817741:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817743:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205251.817745:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205251.817747:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205251.817748:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205251.817749:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817751:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59232:59232:256:4294967295] 192.168.201.151@tcp LPNI seq info [59212:59212:8:4294967295] 00000400:00000200:1.0:1788205251.817754:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817758:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205251.817760:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817763:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205251.817765:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205251.817767:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205251.817769:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205251.817771:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205251.817773:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817774:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205251.817776:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205251.817778:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205251.817781:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205251.817782:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205251.817785:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205251.817788:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817795:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.817798:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817803:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817805:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205251.817808:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205251.817811:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205251.817814:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205251.817816:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301379113 00000100:00000001:1.0:1788205251.817818:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.817821:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301379113 00000100:00000001:1.0:1788205251.817823:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.817824:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205251.817826:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.817828:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205251.817831:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205251/real 1788205251] req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205251.817836:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205251.817837:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205251.817838:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205251.817841:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205251.817843:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205251.817846:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205251.817849:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205251.817856:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817862:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205251.817864:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817865:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817867:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205251.817870:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817878:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205251.817882:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817887:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205251.817889:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817890:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.817891:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.817892:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205251.817894:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205251.817895:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205251.817897:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205251.817899:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205251.817901:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205251.817905:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105edc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944192:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205251.817907:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205251.817910:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817916:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205251.817917:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817918:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205251.817920:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205251.817922:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205251.817925:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205251.817928:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205251.817929:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205251.817931:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205251.817932:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205251.817935:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205251.817937:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205251.817940:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205251.817942:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205251.817945:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205251.817948:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944320, portal 4 00000100:00000001:1.0:1788205251.817950:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205251.817951:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562293536000 : -113511416015616 : ffff98c3105ed500) 00000100:00000040:1.0:1788205251.817955:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.817959:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205251.817960:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205251.817961:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205251.817963:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205944320, offset 0 00000400:00000200:1.0:1788205251.817965:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817967:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817969:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205251.817971:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205251.817973:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205251.817974:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205251.817976:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205251.817978:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59233:59233:256:4294967295] 192.168.201.151@tcp LPNI seq info [59213:59213:8:4294967295] 00000400:00000200:1.0:1788205251.817980:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205251.817984:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205251.817986:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817988:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205251.817990:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205251.817992:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205251.817995:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205251.817996:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205251.817998:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205251.817999:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205251.818001:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205251.818003:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205251.818005:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205251.818006:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205251.818008:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205251.818010:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818017:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.818019:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205276 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818023:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.818023:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205251.818025:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205251.818027:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205251.818028:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205251.818029:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301379119 00000100:00000001:1.0:1788205251.818031:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.818032:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301379119 00000100:00000001:1.0:1788205251.818034:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301383169 00000100:00000001:1.0:1788205251.818036:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205251.818038:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.818040:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205251.818043:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205251/real 1788205251] req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205251.818050:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205251.818051:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205251.818052:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205251.818056:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205251.818057:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205251.818059:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205251.818062:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205251.818068:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818074:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205251.818075:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818077:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.818078:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205251.818081:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818086:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205251.818089:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818095:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205251.818097:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818098:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205251.818099:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818101:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205251.818102:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205251.818105:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205251.818108:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205251.818110:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205251.818120:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205251.818126:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c3105ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944320:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205251.818130:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818131:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.818134:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ef480 x1875062205944064/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818139:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205251.818141:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205251.818142:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205251.818144:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205251.818146:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205251.818147:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205251.818149:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205251.818151:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dc8c00. 02000000:00000001:1.0:1788205251.818153:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205251.818154:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205251.818155:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ef480. 00000100:00000001:1.0:1788205251.818158:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205251.818159:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818160:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.818162:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105edc00 x1875062205944192/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818167:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205251.818168:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205251.818169:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205251.818171:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205251.818172:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205251.818183:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205251.818185:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205251.818186:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca800. 02000000:00000001:1.0:1788205251.818188:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205251.818190:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205251.818191:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105edc00. 00000100:00000001:1.0:1788205251.818193:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205251.818194:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818195:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205251.818198:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c3105ed500 x1875062205944320/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205276 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205251.818203:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205251.818204:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205251.818205:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205251.818207:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205251.818208:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205251.818209:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205251.818210:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205251.818212:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310dca400. 02000000:00000001:1.0:1788205251.818213:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205251.818214:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205251.818215:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c3105ed500. 00000100:00000001:1.0:1788205251.818218:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205251.818219:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818220:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205251.818224:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205251.818225:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205251.818227:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205251.818228:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205252.008503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205252.008509:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205252.008511:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205252.008513:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008515:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205252.008517:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008518:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008519:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205252.008522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205252.008523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205252.008524:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205252.008575:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205252.008577:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205252.008579:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205252.328427:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205252.328434:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205252.328436:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205252.328442:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205252.328446:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205252.328449:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.328452:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205252.328454:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.328456:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205252.776464:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205252.776471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205252.776473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205252.776477:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205252.776480:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205252.776484:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205252.776486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205252.776489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205252.776492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205252.840519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205252.840524:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205252.840530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205252.840532:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205252.840538:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205252.840538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205252.840551:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:2.0:1788205252.840552:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:2.0:1788205252.840561:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205252.840561:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205252.840577:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.840578:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205252.840588:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205252.840592:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205252.840595:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205252.840598:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205252.840603:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.840607:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205253.032458:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205253.032463:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205253.032467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205253.032468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205253.032470:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205253.032471:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205253.032473:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205253.032475:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205253.032527:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205253.032530:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205253.032534:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205253.352460:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205253.352468:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205253.352469:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205253.352475:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205253.352479:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205253.352483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205253.352486:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205253.352489:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205253.352492:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.800455:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205253.800466:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205253.800468:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205253.800472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205253.800475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205253.800478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.800479:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205253.800481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.800483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.864494:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205253.864501:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205253.864503:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205253.864507:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205253.864510:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205253.864515:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205253.864517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205253.864519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205253.864520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205253.864523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205253.864526:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205253.864528:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205253.864533:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205253.864535:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205253.864538:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205253.864541:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205253.864543:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205253.864546:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205254.056530:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205254.056533:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205254.056536:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056540:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205254.056542:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056544:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056547:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056551:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205254.056552:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205254.056556:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205254.056557:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205254.056559:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056560:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205254.056562:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056563:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056565:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056568:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205254.056569:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205254.056571:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205254.056572:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056574:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205254.056575:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056576:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205254.056578:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205254.056630:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205254.056637:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205254.056641:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205254.379631:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205254.379639:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205254.379642:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205254.379647:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:1.0:1788205254.379651:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205254.379654:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205254.379657:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205254.379659:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205254.379662:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205254.824472:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205254.824479:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205254.824481:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205254.824486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205254.824489:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205254.824493:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205254.824496:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205254.824498:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205254.824500:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205254.888491:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205254.888499:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205254.888501:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205254.888505:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205254.888508:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205254.888511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205254.888513:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205254.888515:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205254.888517:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205254.888779:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205254.888782:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205254.888783:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205254.888786:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205254.888789:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205254.888791:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205254.888793:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205254.888794:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205254.888796:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205255.080429:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205255.080435:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205255.080441:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205255.080442:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205255.080445:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205255.080446:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205255.080449:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205255.080451:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205255.080499:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205255.080503:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205255.080507:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205255.400459:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205255.400465:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205255.400466:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205255.400470:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:1.0:1788205255.400472:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205255.400475:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205255.400477:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205255.400479:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205255.400481:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205255.848461:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205255.848468:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205255.848469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205255.848473:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205255.848476:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205255.848480:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205255.848482:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205255.848484:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205255.848487:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205255.912506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205255.912514:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205255.912517:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205255.912522:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205255.912526:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205255.912530:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205255.912533:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205255.912535:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205255.912539:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205255.912554:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205255.912561:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205255.912563:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205255.912569:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205255.912574:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205255.912578:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205255.912582:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205255.912585:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205255.912588:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205256.104507:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205256.104512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205256.104516:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205256.104517:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205256.104519:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205256.104519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205256.104521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205256.104522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205256.104578:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205256.104581:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205256.104584:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205256.424477:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301383169, at 4301383680 00000100:00000040:1.0:1788205256.424477:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301383169, at 4301383680 00000100:00000010:1.0:1788205256.424493:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1840. 00000100:00000010:0.0:1788205256.424498:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1f00. 00000100:00000001:3.0:1788205256.424512:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205256.424517:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205256.424520:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205256.424523:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205256.424526:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205256.424535:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205256.424538:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205256.424540:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205256.424542:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.872496:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205256.872511:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205256.872514:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205256.872521:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205256.872525:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205256.872529:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.872531:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205256.872534:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.872537:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205256.936519:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205256.936520:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205256.936532:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205256.936537:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205256.936538:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205256.936542:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205256.936544:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788205256.936545:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205256.936547:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205256.936554:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205256.936555:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205256.936556:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788205256.936565:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205256.936566:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205256.936567:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205256.936576:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205256.936578:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205256.936582:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205256.936582:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205256.936588:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6711 00000100:00000001:0.0:1788205256.936600:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205256.936600:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205256.936606:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205256.936611:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205256.936613:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205256.936622:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205256.936628:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205256.936633:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936636:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205256.936639:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936641:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205256.936644:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205256.936646:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205256.936649:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205256.936653:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205256.936655:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205256.936657:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205256.936659:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936660:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205256.936661:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936662:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205256.936665:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205256.936668:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936670:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205256.936674:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205256.936678:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205256.936681:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936683:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205256.936686:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205256.936698:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000020:00000040:2.0:1788205256.936702:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205256.936707:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205256.936709:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936712:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205256.936716:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205256.936721:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205256.936731:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205256.936740:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205256.936759:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205256.936769:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205256.936772:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205256.936776:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205256.936802:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205256.936805:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205256.936809:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205256.936810:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205256.936812:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205256.936816:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205256.936818:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6711 00000400:00000200:2.0:1788205256.936823:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205256.936836:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205256.936839:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205256.936841:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936843:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205256.936844:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936846:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205256.936848:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205256.936850:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205256.936851:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205256.936852:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205256.936854:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:1.0:1788205256.936855:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205256.936856:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205256.936861:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205256.936862:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205256.936863:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205256.936867:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205256.936869:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205256.936870:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205256.936875:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205256.936878:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205256.936880:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205256.936881:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205256.936886:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205256.936888:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205256.936889:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936893:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205256.936898:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205256.936902:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936907:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205256.936913:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000100:00000001:1.0:1788205256.936916:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788205256.936917:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000001:1.0:1788205256.936918:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205256.936924:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205256.936926:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205256.936927:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936930:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205256.936933:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205256.936934:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205256.936935:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205256.936941:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 02000000:00000001:1.0:1788205256.936942:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936946:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205256.936952:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205944448:192.168.201.151@tcp:250:kworker.0 00000100:00100000:2.0:1788205256.936954:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205256.936972:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205256.936974:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205256.936979:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.936982:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1788205256.936984:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205256.936988:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.936995:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.936996:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788205256.937003:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205256.937006:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205256.937007:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205256.937008:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205256.937011:0:30130:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1788205256.937012:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bdc00. 00000100:00080000:2.0:1788205256.937016:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6711 02000000:00000001:1.0:1788205256.937020:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205256.937021:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205256.937027:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205256.937029:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205256.937033:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937035:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205256.937039:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:2.0:1788205256.937043:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937046:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000010:1.0:1788205256.937047:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000020:00000001:2.0:1788205256.937056:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205256.937058:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205256.937060:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:1.0:1788205256.937061:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000020:00000040:2.0:1788205256.937062:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205256.937072:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:1.0:1788205256.937077:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944448, portal 25 00000020:00000001:2.0:1788205256.937077:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205256.937102:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937104:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205256.937108:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205256.937110:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205256.937111:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788205256.937117:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000020:00000040:2.0:1788205256.937118:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205256.937125:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937128:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205256.937129:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205256.937133:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205256.937136:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205256.937137:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937146:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205256.937150:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205256.937159:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937169:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937170:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:2.0:1788205256.937176:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000100:00000040:1.0:1788205256.937179:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000020:00000040:2.0:1788205256.937183:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000400:00000010:1.0:1788205256.937186:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 02000000:00000001:2.0:1788205256.937191:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1788205256.937192:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205944448, offset 0 02000000:00000001:2.0:1788205256.937194:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937206:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205256.937209:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205256.937211:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.937217:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 02000000:00000010:2.0:1788205256.937221:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205256.937227:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.937228:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00100000:2.0:1788205256.937236:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205256.937243:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:2.0:1788205256.937257:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205256.937259:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788205256.937268:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.937274:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00000040:2.0:1788205256.937275:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205256.937285:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.937286:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788205256.937288:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205256.937294:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1788205256.937295:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937297:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205256.937301:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205256.937304:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205256.937311:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6711 00000400:00000200:1.0:1788205256.937313:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59234:59234:256:4294967295] 192.168.201.151@tcp LPNI seq info [59214:59214:8:4294967295] 00000400:00000200:2.0:1788205256.937326:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.937334:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205256.937339:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205256.937343:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205256.937349:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937358:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205256.937397:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205256.937404:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.937409:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000100:00000040:2.0:1788205256.937411:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937419:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000800:00000200:1.0:1788205256.937420:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205256.937426:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1788205256.937429:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205256.937434:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000800:00000200:1.1:1788205256.937435:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000020:00000001:2.0:1788205256.937444:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:1.1:1788205256.937445:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000020:00000040:2.0:1788205256.937450:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205256.937459:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000800:00000100:1.0:1788205256.937461:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.937466:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000020:00000001:2.0:1788205256.937468:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000400:00000200:1.0:1788205256.937475:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788205256.937477:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937483:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205256.937489:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788205256.937490:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205256.937498:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:1.0:1788205256.937501:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000020:00000040:2.0:1788205256.937504:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000010:1.0:1788205256.937507:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00080000:2.0:1788205256.937513:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205256.937516:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788205256.937518:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205256.937523:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205256.937526:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205256.937529:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205256.937533:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205256.937534:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205256.937538:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205256.937546:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937553:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205256.937565:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000100:00000001:1.0:1788205256.937567:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1788205256.937572:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000040:1.0:1788205256.937579:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205256.937580:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205256.937583:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937590:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205256.937593:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205256.937595:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205256.937600:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c000. 00000100:00000001:2.0:1788205256.937605:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205256.937618:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205256.937619:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937628:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205256.937638:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205256.937639:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.937642:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205256.937651:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205256.937652:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205256.937659:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301388290 00000100:00000010:1.0:1788205256.937672:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1840. 00000100:00000001:2.0:1788205256.937674:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937681:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1788205256.937684:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6721) 00000100:00000001:1.0:1788205256.937688:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301388290 00000100:00000010:1.0:1788205256.937694:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1f00. 00000100:00000001:1.0:1788205256.937697:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205256.937700:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205256.937702:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937705:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205256.937710:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205256/real 1788205256] req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205256.937717:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205256.937718:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205256.937719:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205256.937721:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205256.937722:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205256.937724:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205256.937727:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205256.937733:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937740:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205256.937741:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937743:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.937745:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205256.937749:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937754:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205256.937759:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937766:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205256.937768:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937770:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.937771:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937774:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205256.937776:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205256.937779:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205256.937782:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205256.937784:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205256.937787:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205256.937792:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205944448:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205256.937795:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937796:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.937798:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205944448/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937806:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205256.937808:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205256.937810:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205256.937812:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205256.937814:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205256.937815:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205256.937817:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205256.937818:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:1.0:1788205256.937822:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205256.937823:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205256.937824:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:1.0:1788205256.937826:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205256.937827:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937829:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.937833:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205256.937834:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937836:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205256.937837:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205256.937838:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205256.937841:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937845:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205256.937846:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.937848:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205256.937849:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205256.937850:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205256.937852:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944576:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205256.937854:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205256.937855:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205256.937857:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.937858:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205256.937860:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205256.937861:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.937863:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205256.937865:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205256.937867:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205256.937869:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944576, portal 10 00000100:00000001:1.0:1788205256.937870:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205256.937871:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:1.0:1788205256.937876:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937880:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205256.937881:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205256.937882:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205256.937884:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205944576, offset 0 00000400:00000200:1.0:1788205256.937886:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205256.937889:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.937891:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205256.937893:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205256.937894:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205256.937896:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205256.937897:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205256.937900:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59235:59235:256:4294967295] 192.168.201.151@tcp LPNI seq info [59215:59215:8:4294967295] 00000400:00000200:1.0:1788205256.937902:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.937905:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205256.937908:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.937910:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205256.937912:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205256.937914:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205256.937917:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205256.937919:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205256.937921:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.937922:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205256.937924:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205256.937926:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205256.937928:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205256.937929:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205256.937931:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205256.937934:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937941:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.937944:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.937949:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937951:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205256.937954:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205256.937957:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205256.937960:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205256.937962:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301384347 00000100:00000001:1.0:1788205256.937964:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.937966:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301384347 00000100:00000001:1.0:1788205256.937968:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.937970:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205256.937972:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.937974:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205256.937977:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205256/real 1788205256] req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205256.937984:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205256.937985:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205256.937986:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205256.937988:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205256.937989:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205256.937990:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205256.937992:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205256.937997:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938004:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205256.938005:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938006:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938007:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205256.938009:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938013:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205256.938015:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938019:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205256.938020:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938021:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938022:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938023:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205256.938025:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205256.938026:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205256.938028:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205256.938029:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205256.938031:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205256.938035:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944576:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205256.938037:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205256.938039:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938043:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205256.938044:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.938046:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205256.938047:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205256.938048:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205256.938050:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944704:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205256.938052:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205256.938053:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205256.938054:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.938055:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205256.938057:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205256.938058:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.938060:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205256.938062:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205256.938064:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205256.938065:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944704, portal 4 00000100:00000001:1.0:1788205256.938067:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205256.938067:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:1.0:1788205256.938071:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938074:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205256.938076:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205256.938077:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205256.938079:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205944704, offset 0 00000400:00000200:1.0:1788205256.938081:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205256.938083:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.938099:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205256.938102:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205256.938104:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205256.938106:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205256.938108:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205256.938111:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59236:59236:256:4294967295] 192.168.201.151@tcp LPNI seq info [59216:59216:8:4294967295] 00000400:00000200:1.0:1788205256.938115:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.938121:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205256.938126:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.938129:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205256.938133:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205256.938135:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205256.938139:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205256.938141:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205256.938143:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.938144:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205256.938147:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205256.938149:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205256.938151:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205256.938152:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205256.938153:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205256.938156:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938160:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.938162:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938166:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.938167:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205256.938169:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205256.938171:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205256.938173:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205256.938174:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301384233 00000100:00000001:1.0:1788205256.938176:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.938177:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301384233 00000100:00000001:1.0:1788205256.938178:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.938179:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205256.938180:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.938181:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205256.938183:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205256/real 1788205256] req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205256.938188:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205256.938189:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205256.938190:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205256.938191:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205256.938192:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205256.938193:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205256.938195:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205256.938200:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938203:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205256.938204:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938205:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938205:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205256.938207:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938211:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205256.938213:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938216:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205256.938218:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938219:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938220:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938221:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205256.938222:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205256.938223:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205256.938225:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205256.938226:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205256.938228:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205256.938233:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944704:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205256.938235:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205256.938237:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938241:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205256.938242:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.938243:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205256.938244:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205256.938245:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205256.938246:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205256.938248:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205256.938249:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205256.938250:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205256.938251:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205256.938253:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205256.938254:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205256.938255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205256.938257:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205256.938258:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205256.938260:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944832, portal 4 00000100:00000001:1.0:1788205256.938261:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205256.938262:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:1.0:1788205256.938266:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938272:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205256.938274:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205256.938276:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205256.938278:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205944832, offset 0 00000400:00000200:1.0:1788205256.938282:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205256.938285:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.938288:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205256.938290:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205256.938293:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205256.938295:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205256.938297:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205256.938300:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59237:59237:256:4294967295] 192.168.201.151@tcp LPNI seq info [59217:59217:8:4294967295] 00000400:00000200:1.0:1788205256.938304:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205256.938310:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205256.938314:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.938317:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205256.938320:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205256.938321:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205256.938324:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205256.938325:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205256.938341:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205256.938343:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205256.938347:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205256.938350:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205256.938353:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205256.938355:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205256.938357:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205256.938360:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938390:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.938393:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205281 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938398:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.938400:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205256.938403:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205256.938406:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205256.938408:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205256.938410:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301384240 00000100:00000001:1.0:1788205256.938412:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.938414:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301384240 00000100:00000001:1.0:1788205256.938416:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301388290 00000100:00000001:1.0:1788205256.938418:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205256.938420:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.938421:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205256.938424:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205256/real 1788205256] req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205256.938431:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205256.938433:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205256.938434:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205256.938436:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205256.938438:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205256.938439:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205256.938441:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205256.938446:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938449:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205256.938450:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938452:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938452:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205256.938454:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938457:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205256.938461:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938467:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205256.938470:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938471:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205256.938472:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938474:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205256.938476:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205256.938478:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205256.938481:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205256.938483:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205256.938487:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205256.938495:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205944832:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205256.938498:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938500:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.938503:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205944576/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938509:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205256.938510:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205256.938511:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205256.938513:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205256.938514:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205256.938515:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205256.938516:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205256.938517:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:1.0:1788205256.938519:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205256.938520:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205256.938521:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:1.0:1788205256.938524:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205256.938525:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938527:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.938530:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205944704/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938540:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205256.938541:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205256.938543:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205256.938545:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205256.938546:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205256.938547:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205256.938549:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205256.938551:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:1.0:1788205256.938553:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205256.938554:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205256.938555:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:1.0:1788205256.938557:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205256.938558:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938560:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205256.938564:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205944832/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205281 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205256.938568:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205256.938569:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205256.938570:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205256.938571:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205256.938572:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205256.938573:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205256.938574:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205256.938575:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c000. 02000000:00000001:1.0:1788205256.938577:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205256.938578:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205256.938579:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:1.0:1788205256.938581:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205256.938582:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938584:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205256.938587:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205256.938588:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205256.938590:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205256.938592:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205257.128450:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205257.128457:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205257.128462:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205257.128464:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205257.128467:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205257.128468:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205257.128471:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205257.128472:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205257.128535:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205257.128543:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205257.128548:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205257.448496:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205257.448503:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205257.448505:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205257.448510:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205257.448513:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205257.448517:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.448520:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205257.448523:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.448525:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205257.896570:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205257.896577:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205257.896579:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205257.896584:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205257.896588:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205257.896591:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205257.896594:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205257.896596:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205257.896599:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205257.960472:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205257.960477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205257.960479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205257.960483:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205257.960485:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205257.960488:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.960490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205257.960491:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.960493:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205257.961233:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205257.961240:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205257.961242:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205257.961247:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205257.961251:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205257.961254:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205257.961256:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205257.961259:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205257.961261:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205258.152563:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205258.152571:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205258.152578:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205258.152579:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205258.152583:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205258.152584:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205258.152589:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205258.152590:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205258.152661:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205258.152666:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205258.152670:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205258.472508:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205258.472516:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205258.472518:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.472524:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205258.472527:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205258.472531:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.472534:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.472537:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.472539:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.920484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205258.920500:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205258.920502:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205258.920508:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205258.920512:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205258.920516:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205258.920518:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205258.920520:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205258.920522:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205258.984500:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205258.984508:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205258.984510:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205258.984515:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205258.984518:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205258.984522:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205258.984525:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205258.984528:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205258.984530:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.986151:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205258.986156:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205258.986158:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.986162:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205258.986165:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205258.986167:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.986169:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.986171:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.986174:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205259.178302:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205259.178307:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205259.178311:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205259.178312:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205259.178314:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205259.178315:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205259.178317:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205259.178318:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205259.178415:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205259.178417:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205259.178419:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205259.496534:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205259.496540:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205259.496541:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205259.496545:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205259.496548:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205259.496550:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205259.496552:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205259.496554:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205259.496555:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205259.944469:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205259.944474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205259.944475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205259.944479:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205259.944482:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205259.944485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205259.944486:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205259.944488:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205259.944490:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205260.008470:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205260.008472:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205260.008477:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205260.008477:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205260.008479:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205260.008481:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205260.008484:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788205260.008486:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205260.008490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205260.008491:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205260.008496:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.008496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.008503:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205260.008507:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205260.008511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205260.008511:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205260.008514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205260.008518:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205260.200508:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205260.200514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205260.200520:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205260.200522:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205260.200524:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205260.200526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205260.200529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205260.200530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205260.200578:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205260.200581:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205260.200584:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205260.520483:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205260.520491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205260.520493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205260.520499:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205260.520503:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205260.520506:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.520509:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205260.520511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.520514:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205260.968488:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205260.968494:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205260.968496:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205260.968500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205260.968502:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205260.968505:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205260.968507:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205260.968509:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205260.968511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205261.032533:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205261.032539:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205261.032542:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205261.032548:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205261.032551:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205261.032555:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205261.032557:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205261.032560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205261.032563:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205261.032744:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205261.032748:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205261.032750:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205261.032752:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205261.032754:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205261.032756:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205261.032758:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205261.032759:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205261.032761:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224479:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205261.224485:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224488:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205261.224491:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205261.224493:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205261.224495:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224497:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205261.224499:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224500:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224502:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205261.224505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205261.224509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205261.224510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205261.224746:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205261.224749:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205261.224752:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205261.544508:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301388290, at 4301388800 00000100:00000001:3.0:1788205261.544509:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205261.544517:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205261.544520:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205261.544524:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1f00. 00000100:00100000:3.0:1788205261.544525:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205261.544530:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205261.544534:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205261.544537:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205261.544540:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205261.544544:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205261.544710:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301388290, at 4301388800 00000100:00000010:0.0:1788205261.544717:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1840. 00000100:00000001:1.0:1788205261.993188:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205261.993202:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205261.993204:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205261.993210:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205261.993214:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205261.993217:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205261.993220:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205261.993222:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205261.993226:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205262.056508:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205262.056520:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205262.056522:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205262.056526:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205262.056531:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205262.056540:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6716 00000400:00000200:2.0:1788205262.056550:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205262.056555:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205262.056558:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205262.056561:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056565:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.056567:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056569:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205262.056573:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205262.056575:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205262.056577:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205262.056581:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205262.056583:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205262.056585:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205262.056588:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056589:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.056591:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056593:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205262.056596:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205262.056600:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056602:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205262.056605:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205262.056609:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205262.056611:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.056613:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205262.056616:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205262.056626:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205262.056629:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205262.056633:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205262.056635:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.056637:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205262.056640:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205262.056644:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205262.056651:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e400. 00000100:00000001:2.0:1788205262.056658:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205262.056667:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205262.056676:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205262.056680:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205262.056684:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205262.056708:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205262.056711:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205262.056714:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205262.056715:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205262.056717:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205262.056720:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205262.056722:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6716 00000400:00000200:2.0:1788205262.056726:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205262.056729:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205262.056731:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205262.056734:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056737:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.056740:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056742:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205262.056744:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205262.056746:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205262.056748:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205262.056750:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205262.056753:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:1.0:1788205262.056755:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205262.056756:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:3.0:1788205262.056759:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205262.056761:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205262.056763:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205262.056765:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205262.056766:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205262.056767:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205262.056770:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205262.056773:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000040:1.0:1788205262.056776:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205262.056777:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:3.0:1788205262.056777:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000020:00000040:2.0:1788205262.056784:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:3.0:1788205262.056784:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205262.056790:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.056791:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205262.056792:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205262.056796:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.056798:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205262.056798:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788205262.056802:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205262.056803:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205262.056803:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205262.056807:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205262.056810:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205262.056810:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.056811:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205262.056817:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205262.056820:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205944960:192.168.201.151@tcp:250:kworker.0 00000100:00000001:2.0:1788205262.056821:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205262.056827:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205262.056828:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788205262.056833:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.056835:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788205262.056836:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.056840:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205262.056842:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 02000000:00000001:1.0:1788205262.056844:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1788205262.056847:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205262.056851:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1788205262.056852:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:2.0:1788205262.056853:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.056857:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.056858:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205262.056861:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205262.056864:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.056868:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788205262.056872:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e000. 00000400:00000010:1.0:1788205262.056875:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000100:00000001:2.0:1788205262.056877:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205262.056883:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00100000:2.0:1788205262.056885:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205262.056892:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205944960, portal 25 00000100:00000001:2.0:1788205262.056895:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205262.056897:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205262.056899:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.056901:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:2.0:1788205262.056906:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205262.056913:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205262.056922:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205262.056925:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205262.056928:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205262.056930:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205262.056931:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205262.056933:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205262.056935:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1788205262.056939:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205262.056941:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6716 00000100:00000001:0.0:1788205262.056943:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205262.056948:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000400:00000200:2.0:1788205262.056949:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205262.056953:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1788205262.056956:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205262.056958:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205262.056960:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205262.056962:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205262.056966:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205262.056966:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205262.056970:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.056971:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.056977:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.056980:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205262.056983:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205262.056984:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205262.056986:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205262.056989:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205262.056991:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205262.056994:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205262.057013:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057015:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.057017:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057020:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205262.057022:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205262.057025:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057027:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205262.057029:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205262.057031:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205262.057033:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.057035:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205262.057037:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:2.0:1788205262.057039:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.057040:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205262.057043:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 00000020:00000040:2.0:1788205262.057046:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205262.057049:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205262.057050:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.057052:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205262.057053:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205262.057055:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205262.057058:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205262.057062:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205262.057066:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205262.057073:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205262.057075:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205262.057079:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205262.057086:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205262.057089:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205262.057092:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205262.057093:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205262.057094:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205262.057097:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205262.057099:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6716 00000400:00000200:2.0:1788205262.057103:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205262.057106:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205262.057108:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205262.057110:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057112:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.057114:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057115:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205262.057117:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205262.057118:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205262.057119:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205262.057121:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205262.057123:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205262.057125:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205262.057128:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057130:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205262.057132:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057135:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205262.057137:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:2.0:1788205262.057139:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057141:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205262.057142:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205262.057144:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205262.057145:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.057147:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205262.057149:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205262.057151:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205262.057153:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205262.057157:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205262.057161:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:2.0:1788205262.057165:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205262.057168:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057170:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205262.057170:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205262.057173:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205262.057175:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205262.057179:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 02000000:00000010:2.0:1788205262.057180:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000400:00000010:1.0:1788205262.057184:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000001:2.0:1788205262.057185:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205262.057193:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205944960, offset 0 00000100:00100000:2.0:1788205262.057193:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205262.057205:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205262.057206:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205262.057209:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205262.057214:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205262.057216:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205262.057225:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.057226:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:2.0:1788205262.057231:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6727) 00000400:00000200:1.0:1788205262.057233:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205262.057239:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205262.057243:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205262.057246:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205262.057251:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59238:59238:256:4294967295] 192.168.201.151@tcp LPNI seq info [59218:59218:8:4294967295] 00000400:00000200:1.0:1788205262.057255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.057265:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205262.057270:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.057275:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205262.057281:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205262.057284:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788205262.057290:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205262.057293:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788205262.057296:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.057299:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205262.057305:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205262.057309:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205262.057314:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205262.057316:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205262.057320:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205262.057324:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057334:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.057338:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057345:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057347:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205262.057352:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.057355:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205262.057358:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301393409 00000100:00000010:1.0:1788205262.057362:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1f00. 00000100:00000001:1.0:1788205262.057392:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205262.057398:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301393410 00000100:00000010:1.0:1788205262.057401:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c15a0. 00000100:00000001:1.0:1788205262.057403:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205262.057406:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205262.057409:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057411:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205262.057416:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205262/real 1788205262] req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205262.057424:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205262.057426:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205262.057427:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205262.057430:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205262.057432:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205262.057434:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205262.057437:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205262.057445:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057451:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205262.057454:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057456:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.057458:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205262.057462:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057468:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205262.057472:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057480:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205262.057482:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057484:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.057485:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057487:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205262.057489:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205262.057492:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205262.057495:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205262.057497:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205262.057500:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205262.057507:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205944960:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205262.057511:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057513:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.057516:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205944960/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057522:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205262.057524:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205262.057526:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205262.057529:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205262.057531:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205262.057533:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205262.057535:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205262.057537:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e400. 02000000:00000001:1.0:1788205262.057541:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205262.057542:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205262.057544:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:1.0:1788205262.057547:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205262.057548:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057551:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057558:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205262.057559:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057562:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205262.057563:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205262.057564:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205262.057568:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057575:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205262.057576:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.057578:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205262.057579:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205262.057582:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205262.057584:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945088:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205262.057587:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205262.057589:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205262.057591:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.057592:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205262.057595:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205262.057597:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.057600:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205262.057603:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205262.057607:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205262.057610:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945088, portal 10 00000100:00000001:1.0:1788205262.057612:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205262.057613:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:1.0:1788205262.057619:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057626:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205262.057628:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205262.057630:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205262.057633:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205945088, offset 0 00000400:00000200:1.0:1788205262.057636:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205262.057640:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.057643:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205262.057646:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205262.057649:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205262.057651:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205262.057653:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205262.057656:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59239:59239:256:4294967295] 192.168.201.151@tcp LPNI seq info [59219:59219:8:4294967295] 00000400:00000200:1.0:1788205262.057660:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.057665:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205262.057670:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.057673:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205262.057677:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205262.057679:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788205262.057684:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205262.057686:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788205262.057690:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.057692:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205262.057695:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205262.057698:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205262.057700:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205262.057702:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205262.057705:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205262.057708:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057715:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.057718:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057730:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057731:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205262.057734:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205262.057738:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205262.057741:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205262.057743:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301389467 00000100:00000001:1.0:1788205262.057746:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301393409 00000100:00000001:1.0:1788205262.057748:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301389467 00000100:00000001:1.0:1788205262.057750:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301393410 00000100:00000001:1.0:1788205262.057752:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205262.057754:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.057756:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205262.057759:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205262/real 1788205262] req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205262.057767:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205262.057768:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205262.057769:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205262.057771:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205262.057773:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205262.057775:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205262.057778:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205262.057786:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057791:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205262.057793:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057794:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.057796:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205262.057799:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057805:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205262.057809:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057815:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205262.057818:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057819:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.057821:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.057822:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205262.057825:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205262.057827:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205262.057831:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205262.057832:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205262.057836:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205262.057843:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945088:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205262.057847:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205262.057850:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057856:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205262.057858:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.057860:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205262.057862:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205262.057864:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205262.057866:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945216:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205262.057869:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205262.057871:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205262.057873:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.057875:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205262.057878:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205262.057879:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.057882:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205262.057885:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205262.057888:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205262.057891:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945216, portal 4 00000100:00000001:1.0:1788205262.057893:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205262.057894:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:1.0:1788205262.057899:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.057906:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205262.057908:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205262.057910:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205262.057912:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205945216, offset 0 00000400:00000200:1.0:1788205262.057916:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205262.057933:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.057937:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205262.057939:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205262.057942:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205262.057944:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205262.057946:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205262.057949:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59240:59240:256:4294967295] 192.168.201.151@tcp LPNI seq info [59220:59220:8:4294967295] 00000400:00000200:1.0:1788205262.057953:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.057958:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205262.057963:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.057966:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205262.057970:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205262.057972:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788205262.057976:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205262.057979:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788205262.057993:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.058004:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205262.058008:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205262.058011:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205262.058014:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205262.058015:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205262.058017:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205262.058020:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058028:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.058030:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058037:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.058038:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205262.058041:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205262.058044:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205262.058047:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205262.058048:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301389353 00000100:00000001:1.0:1788205262.058051:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301393409 00000100:00000001:1.0:1788205262.058052:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301389353 00000100:00000001:1.0:1788205262.058054:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301393410 00000100:00000001:1.0:1788205262.058057:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205262.058058:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.058060:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205262.058064:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205262/real 1788205262] req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205262.058071:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205262.058073:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205262.058075:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205262.058077:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205262.058078:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205262.058081:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205262.058084:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205262.058092:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058099:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205262.058100:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058102:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.058103:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205262.058107:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058114:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205262.058118:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058124:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205262.058127:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058128:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.058129:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058131:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205262.058133:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205262.058136:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205262.058139:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205262.058141:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205262.058145:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205262.058153:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945216:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205262.058157:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205262.058160:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058167:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205262.058168:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.058171:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205262.058172:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205262.058174:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205262.058178:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945344:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205262.058181:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205262.058183:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205262.058184:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205262.058187:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205262.058190:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205262.058192:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205262.058195:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205262.058197:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205262.058200:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205262.058203:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945344, portal 4 00000100:00000001:1.0:1788205262.058205:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205262.058206:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000040:1.0:1788205262.058212:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058217:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205262.058219:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205262.058221:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205262.058224:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205945344, offset 0 00000400:00000200:1.0:1788205262.058227:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205262.058231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.058235:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205262.058238:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205262.058240:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205262.058243:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205262.058245:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205262.058248:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59241:59241:256:4294967295] 192.168.201.151@tcp LPNI seq info [59221:59221:8:4294967295] 00000400:00000200:1.0:1788205262.058252:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205262.058258:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205262.058263:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.058266:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205262.058270:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205262.058272:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788205262.058277:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205262.058279:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788205262.058283:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205262.058284:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205262.058287:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205262.058290:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205262.058293:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205262.058295:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205262.058297:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205262.058300:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058308:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.058311:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058317:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.058319:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205262.058322:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205262.058324:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205262.058327:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205262.058329:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301389359 00000100:00000001:1.0:1788205262.058331:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301393409 00000100:00000001:1.0:1788205262.058334:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301389359 00000100:00000001:1.0:1788205262.058336:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301393410 00000100:00000001:1.0:1788205262.058338:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205262.058340:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.058342:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205262.058346:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205262/real 1788205262] req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205262.058353:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205262.058355:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205262.058356:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205262.058358:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205262.058360:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205262.058383:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205262.058387:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205262.058395:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058401:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205262.058402:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058404:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.058405:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205262.058409:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058416:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205262.058419:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058426:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205262.058428:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058429:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205262.058430:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058432:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205262.058434:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205262.058436:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205262.058440:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205262.058442:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205262.058445:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205262.058452:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945344:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205262.058455:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058457:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.058460:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205945088/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058466:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205262.058467:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205262.058469:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205262.058471:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205262.058473:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205262.058474:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205262.058476:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205262.058478:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e000. 02000000:00000001:1.0:1788205262.058481:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205262.058482:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205262.058483:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:1.0:1788205262.058486:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205262.058487:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058488:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.058491:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205945216/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058497:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205262.058498:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205262.058500:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205262.058502:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205262.058503:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205262.058504:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205262.058506:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205262.058508:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:1.0:1788205262.058510:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205262.058511:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205262.058512:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:1.0:1788205262.058514:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205262.058516:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058517:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205262.058520:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205945344/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205262.058526:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205262.058528:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205262.058529:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205262.058531:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205262.058532:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205262.058533:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205262.058535:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205262.058537:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:1.0:1788205262.058539:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205262.058540:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205262.058541:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:1.0:1788205262.058543:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205262.058544:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058547:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205262.058551:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205262.058553:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205262.058555:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205262.058556:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205262.248529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205262.248535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205262.248540:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205262.248542:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205262.248545:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205262.248546:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205262.248549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205262.248550:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205262.251706:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205262.251711:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205262.251714:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205262.568476:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205262.568484:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205262.568486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205262.568491:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205262.568495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205262.568498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205262.568501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205262.568503:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205262.568506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.017355:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205263.017392:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205263.017395:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.017401:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205263.017404:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205263.017408:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205263.017410:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205263.017413:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205263.017416:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205263.080503:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205263.080511:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205263.080513:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205263.080518:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205263.080522:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205263.080526:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205263.080529:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205263.080531:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205263.080534:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.081386:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205263.081390:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205263.081391:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.081394:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205263.081397:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205263.081399:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205263.081401:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205263.081403:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205263.081404:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205263.273078:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205263.273085:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205263.273090:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205263.273092:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205263.273094:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205263.273095:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205263.273099:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205263.273100:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205263.273199:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205263.273201:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205263.273204:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205263.593544:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205263.593552:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205263.593554:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.593560:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205263.593564:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205263.593568:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.593570:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.593573:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.593576:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205264.040491:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205264.040498:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205264.040500:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205264.040505:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205264.040508:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205264.040512:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205264.040514:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205264.040516:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205264.040518:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205264.104619:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205264.104627:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205264.104629:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205264.104634:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205264.104638:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205264.104640:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205264.104643:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205264.104646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205264.104648:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205264.105073:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205264.105076:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205264.105078:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205264.105081:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205264.105083:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205264.105085:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.105087:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205264.105089:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.105091:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205264.296534:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205264.296537:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205264.296540:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296545:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205264.296548:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296549:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296551:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296555:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205264.296556:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205264.296561:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205264.296562:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205264.296564:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296565:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205264.296567:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296568:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296570:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296573:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205264.296575:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205264.296576:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205264.296577:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296579:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205264.296580:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296581:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205264.296583:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205264.296647:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205264.296651:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205264.296654:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205264.616474:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205264.616482:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205264.616484:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205264.616489:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205264.616491:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205264.616494:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.616497:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205264.616499:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.616502:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205265.064453:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205265.064463:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205265.064466:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205265.064473:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205265.064477:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205265.064482:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205265.064485:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205265.064489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205265.064492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205265.128508:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205265.128519:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205265.128520:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205265.128524:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205265.128528:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205265.128531:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205265.128535:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788205265.128537:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205265.128543:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205265.128545:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205265.128549:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205265.128551:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.128555:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205265.128560:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.128564:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205265.128565:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205265.128570:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205265.128573:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205265.320501:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205265.320510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205265.320517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205265.320519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205265.320523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205265.320525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205265.320529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205265.320530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205265.320631:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205265.320638:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205265.320643:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205265.640482:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205265.640491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205265.640493:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205265.640497:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205265.640500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205265.640505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.640507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205265.640509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.640512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205266.088477:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205266.088484:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205266.088486:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205266.088490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205266.088493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205266.088498:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205266.088500:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205266.088502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205266.088504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205266.152520:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205266.152520:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205266.152529:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205266.152530:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205266.152533:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205266.152539:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205266.152541:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205266.152545:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205266.152548:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205266.152550:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205266.152554:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.152555:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.152564:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205266.152567:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.152570:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205266.152574:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205266.152579:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205266.152582:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205266.344553:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205266.344564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205266.344571:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205266.344573:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205266.344576:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205266.344577:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205266.344581:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205266.344582:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205266.344671:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205266.344679:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205266.344684:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205266.664523:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301393410, at 4301393920 00000100:00000001:3.0:1788205266.664523:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205266.664532:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205266.664536:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205266.664539:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c15a0. 00000100:00100000:3.0:1788205266.664541:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205266.664547:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205266.664551:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.664554:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205266.664558:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.664562:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205266.665045:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301393409, at 4301393920 00000100:00000010:0.0:1788205266.665059:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1f00. 00000100:00000001:1.0:1788205267.112470:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205267.112475:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205267.112477:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.112480:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205267.112483:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205267.112486:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.112488:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.112489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.112491:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205267.176508:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205267.176517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205267.176519:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205267.176524:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205267.176528:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205267.176532:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205267.176535:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205267.176538:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205267.176541:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.176741:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205267.176744:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205267.176745:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.176748:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205267.176750:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205267.176752:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.176754:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.176755:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.176756:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205267.176778:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205267.176784:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205267.176787:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205267.177034:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205267.177039:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205267.177046:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6722 00000400:00000200:3.0:1788205267.177055:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205267.177060:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205267.177063:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205267.177066:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177069:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177072:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177074:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205267.177078:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205267.177080:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:3.0:1788205267.177083:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205267.177086:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:3.0:1788205267.177088:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:3.0:1788205267.177090:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:3.0:1788205267.177093:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177095:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177097:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177099:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205267.177102:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:3.0:1788205267.177106:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177108:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205267.177111:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205267.177114:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205267.177117:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177119:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205267.177122:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205267.177131:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0000. 00000020:00000040:3.0:1788205267.177134:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:3.0:1788205267.177138:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205267.177140:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177142:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205267.177145:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205267.177148:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205267.177156:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23800. 00000100:00000001:3.0:1788205267.177162:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205267.177172:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205267.177183:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205267.177187:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205267.177191:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee757f9f] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205267.177216:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205267.177220:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205267.177223:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205267.177225:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205267.177226:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205267.177229:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205267.177232:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6722 00000400:00000200:3.0:1788205267.177236:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205267.177239:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205267.177241:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205267.177244:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177246:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177247:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177249:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205267.177251:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205267.177253:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:3.0:1788205267.177254:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205267.177256:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:3.0:1788205267.177258:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:3.0:1788205267.177260:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:3.0:1788205267.177263:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177264:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177266:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177268:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205267.177270:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:3.0:1788205267.177273:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177275:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205267.177277:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205267.177283:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205267.177285:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177287:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205267.177291:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205267.177294:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177296:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205267.177300:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1880. 00000020:00000040:3.0:1788205267.177302:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:3.0:1788205267.177304:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205267.177306:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177308:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205267.177309:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205267.177311:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205267.177316:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22c00. 00000100:00000001:3.0:1788205267.177320:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205267.177324:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205267.177331:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205267.177333:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205267.177337:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072ac1564] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205267.177357:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205267.177359:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205267.177379:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205267.177381:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205267.177383:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205267.177386:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205267.177388:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6722 00000400:00000200:3.0:1788205267.177392:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205267.177395:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205267.177397:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205267.177400:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177402:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177404:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177406:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205267.177409:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205267.177410:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:3.0:1788205267.177411:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205267.177413:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:3.0:1788205267.177415:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:3.0:1788205267.177418:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:3.0:1788205267.177421:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177423:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177425:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177427:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205267.177429:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:3.0:1788205267.177432:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177434:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205267.177436:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205267.177438:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205267.177441:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177442:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205267.177445:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205267.177447:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177448:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205267.177452:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1180. 00000020:00000040:3.0:1788205267.177454:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205267.177456:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205267.177458:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177459:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205267.177461:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205267.177462:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205267.177466:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21000. 00000100:00000001:3.0:1788205267.177469:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205267.177474:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205267.177481:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205267.177483:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205267.177487:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003da94af2] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205267.177494:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205267.177497:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205267.177500:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205267.177501:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205267.177502:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205267.177505:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205267.177507:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6722 00000400:00000200:3.0:1788205267.177510:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205267.177513:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205267.177515:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205267.177518:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177520:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177522:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177524:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205267.177526:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205267.177527:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205267.177529:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205267.177531:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205267.177533:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205267.177535:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205267.177538:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177540:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205267.177542:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177543:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205267.177545:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205267.177548:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205267.177550:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205267.177551:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205267.177554:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205267.177556:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177558:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205267.177560:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788205267.177562:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177564:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205267.177567:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce1f80. 00000020:00000040:3.0:1788205267.177570:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205267.177572:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205267.177573:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.177575:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205267.177576:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205267.177578:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205267.177581:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c21c00. 00000100:00000001:3.0:1788205267.177584:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205267.177589:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205267.177596:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205267.177598:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205267.177601:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002399dd83] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205267.177608:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205267.177612:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6732) 00000100:00000001:1.0:1788205267.177660:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205267.177665:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205267.177667:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205267.177672:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.177680:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.177682:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.177685:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205267.177688:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205267.177690:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.177693:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205945472:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205267.177698:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205267.177702:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205267.177704:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.177706:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205267.177711:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205267.177713:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205267.177718:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205267.177722:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205267.177729:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205267.177736:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945472, portal 25 00000100:00000001:1.0:1788205267.177739:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205267.177740:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300821504 : -113511408730112 : ffff98c310ce0000) 00000100:00000040:1.0:1788205267.177746:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.177753:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205267.177755:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205267.177758:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205267.177761:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205945472, offset 0 00000400:00000200:1.0:1788205267.177768:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205267.177777:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.177784:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205267.177790:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205267.177794:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205267.177798:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205267.177801:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205267.177805:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59242:59242:256:4294967295] 192.168.201.151@tcp LPNI seq info [59222:59222:8:4294967295] 00000400:00000200:1.0:1788205267.177809:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.177818:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205267.177825:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.177830:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205267.177837:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205267.177841:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788205267.177846:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205267.177850:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1788205267.177854:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.177857:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205267.177864:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205267.177868:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205267.177874:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205267.177876:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205267.177879:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205267.177883:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.177891:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.177894:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.177901:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.177902:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205267.177919:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205267.177923:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205267.177926:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301398530 00000100:00000010:1.0:1788205267.177932:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c15a0. 00000100:00000001:1.0:1788205267.177939:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205267.177944:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301398530 00000100:00000010:1.0:1788205267.177948:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1ea0. 00000100:00000001:1.0:1788205267.177951:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205267.177954:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205267.177957:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.177959:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205267.177964:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205267/real 1788205267] req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205267.177972:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205267.177974:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205267.177976:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205267.177979:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205267.177980:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205267.177983:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205267.177986:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205267.177994:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178000:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205267.178002:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178004:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.178006:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205267.178010:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178017:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205267.178021:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178029:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205267.178031:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178033:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.178034:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178037:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205267.178039:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205267.178041:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205267.178045:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205267.178048:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205267.178052:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205267.178061:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205945472:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205267.178065:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205267.178068:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178073:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.178075:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.178077:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205267.178079:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205267.178080:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.178083:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945600:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205267.178086:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205267.178088:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205267.178089:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.178091:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205267.178094:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205267.178096:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205267.178099:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205267.178101:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205267.178104:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205267.178107:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945600, portal 10 00000100:00000001:1.0:1788205267.178108:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205267.178110:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300827776 : -113511408723840 : ffff98c310ce1880) 00000100:00000040:1.0:1788205267.178115:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178122:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205267.178124:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205267.178126:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205267.178127:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205945600, offset 0 00000400:00000200:1.0:1788205267.178131:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205267.178134:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.178137:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205267.178140:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205267.178142:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205267.178144:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205267.178146:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205267.178149:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59243:59243:256:4294967295] 192.168.201.151@tcp LPNI seq info [59223:59223:8:4294967295] 00000400:00000200:1.0:1788205267.178153:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.178158:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205267.178162:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.178166:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205267.178170:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205267.178172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788205267.178176:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205267.178178:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1788205267.178182:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.178183:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205267.178186:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205267.178189:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205267.178192:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205267.178194:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205267.178196:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205267.178199:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178205:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.178208:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178219:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.178220:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205267.178223:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205267.178226:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205267.178229:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205267.178231:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301394587 00000100:00000001:1.0:1788205267.178233:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.178235:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301394587 00000100:00000001:1.0:1788205267.178237:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.178239:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205267.178241:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.178242:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205267.178245:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205267/real 1788205267] req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205267.178251:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205267.178252:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205267.178254:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205267.178256:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205267.178257:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205267.178259:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205267.178262:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205267.178269:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178276:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205267.178277:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178279:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.178280:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205267.178283:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178290:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205267.178293:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178299:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205267.178302:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178303:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.178304:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.178306:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205267.178308:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205267.178310:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205267.178313:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205267.178315:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205267.178319:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205267.178326:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945600:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205267.178330:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205267.178333:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178339:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.178340:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.178342:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205267.178344:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205267.178345:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.178348:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945728:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205267.178351:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205267.178353:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205267.178354:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.178356:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205267.178359:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205267.178361:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205267.178398:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205267.178402:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205267.178406:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205267.178409:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945728, portal 4 00000100:00000001:1.0:1788205267.178411:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205267.178412:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300825984 : -113511408725632 : ffff98c310ce1180) 00000100:00000040:1.0:1788205267.178418:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.178424:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205267.178427:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205267.178429:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205267.178431:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205945728, offset 0 00000400:00000200:1.0:1788205267.178435:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205267.179009:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.179015:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205267.179017:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205267.179020:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205267.179022:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205267.179025:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205267.179028:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59244:59244:256:4294967295] 192.168.201.151@tcp LPNI seq info [59224:59224:8:4294967295] 00000400:00000200:1.0:1788205267.179032:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.179038:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205267.179043:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.179047:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205267.179052:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205267.179054:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788205267.179058:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205267.179061:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1788205267.179065:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.179068:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205267.179071:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205267.179074:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205267.179077:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205267.179079:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205267.179081:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205267.179085:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179093:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179096:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179102:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.179103:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205267.179106:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205267.179109:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205267.179112:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205267.179114:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301394474 00000100:00000001:1.0:1788205267.179117:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.179119:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301394474 00000100:00000001:1.0:1788205267.179121:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.179123:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205267.179124:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.179126:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205267.179130:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205267/real 1788205267] req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205267.179136:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205267.179138:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205267.179139:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205267.179141:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205267.179143:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205267.179145:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205267.179148:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205267.179156:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179162:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205267.179164:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179165:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.179166:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205267.179170:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179177:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205267.179180:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179187:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205267.179189:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179191:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.179192:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179194:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205267.179196:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205267.179199:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205267.179202:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205267.179204:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205267.179208:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205267.179215:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945728:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205267.179219:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205267.179222:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179229:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205267.179230:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.179232:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205267.179234:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205267.179236:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205267.179239:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205267.179242:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205267.179243:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205267.179245:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205267.179247:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205267.179250:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205267.179252:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205267.179255:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205267.179257:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205267.179260:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205267.179262:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945856, portal 4 00000100:00000001:1.0:1788205267.179264:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205267.179265:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300829568 : -113511408722048 : ffff98c310ce1f80) 00000100:00000040:1.0:1788205267.179270:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179275:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205267.179277:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205267.179279:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205267.179281:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205945856, offset 0 00000400:00000200:1.0:1788205267.179285:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205267.179288:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.179292:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205267.179294:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205267.179297:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205267.179299:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205267.179301:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205267.179304:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59245:59245:256:4294967295] 192.168.201.151@tcp LPNI seq info [59225:59225:8:4294967295] 00000400:00000200:1.0:1788205267.179308:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205267.179313:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205267.179317:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.179320:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205267.179323:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205267.179326:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788205267.179330:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205267.179332:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1788205267.179336:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205267.179337:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205267.179340:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205267.179343:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205267.179346:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205267.179347:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205267.179350:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205267.179353:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179360:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179411:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179418:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.179420:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205267.179423:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205267.179426:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205267.179432:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205267.179434:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301394481 00000100:00000001:1.0:1788205267.179436:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.179438:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301394481 00000100:00000001:1.0:1788205267.179440:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301398530 00000100:00000001:1.0:1788205267.179442:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205267.179444:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.179446:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205267.179450:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205267/real 1788205267] req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205267.179457:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205267.179458:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205267.179459:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205267.179461:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205267.179462:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205267.179464:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205267.179467:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205267.179473:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179479:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205267.179480:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179481:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.179482:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205267.179485:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179491:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205267.179494:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179500:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205267.179503:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179504:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205267.179505:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179507:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205267.179509:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205267.179511:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205267.179515:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205267.179516:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205267.179520:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205267.179527:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205945856:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205267.179531:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179533:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179537:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0000 x1875062205945472/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179543:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205267.179546:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205267.179548:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:1.0:1788205267.179551:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205267.179553:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205267.179554:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205267.179557:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205267.179559:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23800. 02000000:00000001:1.0:1788205267.179562:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205267.179564:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205267.179566:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0000. 00000100:00000001:1.0:1788205267.179568:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205267.179569:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179571:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179574:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1880 x1875062205945600/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179579:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205267.179580:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205267.179583:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205267.179585:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205267.179586:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205267.179588:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205267.179589:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205267.179591:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22c00. 02000000:00000001:1.0:1788205267.179593:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205267.179594:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205267.179595:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1880. 00000100:00000001:1.0:1788205267.179597:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205267.179598:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179599:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179602:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1180 x1875062205945728/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179607:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205267.179608:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205267.179609:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205267.179611:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205267.179612:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205267.179613:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205267.179615:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205267.179616:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21000. 02000000:00000001:1.0:1788205267.179618:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205267.179619:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205267.179620:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1180. 00000100:00000001:1.0:1788205267.179622:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205267.179623:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179625:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205267.179628:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce1f80 x1875062205945856/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205267.179633:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205267.179634:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205267.179635:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205267.179637:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205267.179638:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205267.179639:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205267.179641:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205267.179642:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c21c00. 02000000:00000001:1.0:1788205267.179644:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205267.179645:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205267.179646:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce1f80. 00000100:00000001:1.0:1788205267.179648:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205267.179649:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179651:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205267.179659:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205267.179661:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205267.179663:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205267.179664:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205267.368824:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205267.368832:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205267.368839:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205267.368840:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205267.368843:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205267.368844:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205267.368847:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205267.368848:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205267.369405:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205267.369408:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205267.369411:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205267.688477:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205267.688484:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205267.688486:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205267.688492:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205267.688495:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205267.688498:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205267.688501:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205267.688504:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205267.688506:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205268.136440:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205268.136446:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205268.136452:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205268.136457:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205268.136460:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205268.136462:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205268.136464:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205268.136465:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205268.136467:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.200476:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205268.200476:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205268.200484:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205268.200484:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205268.200489:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205268.200489:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205268.200496:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:1.0:1788205268.200496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205268.200502:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205268.200502:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205268.200507:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205268.200508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205268.200511:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205268.200514:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205268.200518:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.200519:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205268.200522:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205268.200525:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205268.392528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205268.392534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205268.392539:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205268.392540:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205268.392543:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205268.392545:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205268.392547:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205268.392549:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205268.392612:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205268.392616:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205268.392619:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205268.712492:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205268.712498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205268.712500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.712505:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205268.712508:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205268.712513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.712516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205268.712519:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.712522:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205269.160570:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205269.160576:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205269.160578:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205269.160582:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205269.160585:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205269.160587:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205269.160589:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205269.160590:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205269.160596:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205269.224543:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205269.224551:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205269.224553:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205269.224559:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205269.224563:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205269.224568:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205269.224570:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205269.224573:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205269.224576:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205269.224713:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205269.224716:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205269.224718:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205269.224720:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205269.224722:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205269.224724:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205269.224726:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205269.224727:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205269.224729:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205269.416480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205269.416487:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205269.416493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205269.416495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205269.416498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205269.416500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205269.416504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205269.416505:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205269.416563:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205269.416567:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205269.416570:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205269.736433:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205269.736439:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205269.736441:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205269.736445:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205269.736448:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205269.736450:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205269.736453:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205269.736454:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205269.736457:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205270.184434:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205270.184440:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205270.184442:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205270.184458:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205270.184462:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205270.184465:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205270.184467:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205270.184468:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205270.184470:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205270.248506:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205270.248516:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205270.248518:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205270.248533:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:1.0:1788205270.248537:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205270.248542:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205270.248545:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205270.248548:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205270.248551:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205270.251244:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205270.251251:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205270.251254:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205270.251259:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205270.251262:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205270.251265:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205270.251267:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205270.251270:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205270.251273:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205270.440570:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205270.440580:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205270.440586:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205270.440587:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205270.440591:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205270.440593:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205270.440597:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205270.440598:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205270.440680:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205270.440686:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205270.440690:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205270.760658:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205270.760665:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205270.760667:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205270.760671:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205270.760675:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205270.760678:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205270.760681:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205270.760684:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205270.760687:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205271.208497:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205271.208506:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205271.208508:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205271.208513:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205271.208517:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205271.208521:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205271.208523:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205271.208526:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205271.208529:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205271.272531:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205271.272539:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205271.272542:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205271.272547:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205271.272551:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205271.272555:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205271.272558:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205271.272560:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205271.272564:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205271.272699:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205271.272703:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205271.272704:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205271.272707:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205271.272710:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205271.272712:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.272714:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205271.272715:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.272717:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205271.464498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464504:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205271.464508:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205271.464511:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205271.464514:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464518:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205271.464520:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464521:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464524:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464527:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205271.464528:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205271.464532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205271.464533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205271.465904:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205271.465908:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205271.465911:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205271.784473:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301398530, at 4301399040 00000100:00000001:3.0:1788205271.784480:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205271.784486:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1788205271.784488:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c15a0. 00000100:00000001:3.0:1788205271.784489:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205271.784496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205271.784499:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205271.784502:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.784505:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205271.784508:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.784511:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205271.784745:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301398530, at 4301399040 00000100:00000010:1.0:1788205271.784753:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1ea0. 00000100:00000001:1.0:1788205272.232440:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205272.232461:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205272.232463:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205272.232468:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205272.232472:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205272.232475:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.232477:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205272.232480:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.232483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205272.296479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205272.296479:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205272.296486:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205272.296486:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205272.296488:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.296490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205272.296494:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205272.296495:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205272.296499:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205272.296500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205272.296503:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205272.296504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205272.296506:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205272.296510:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205272.296511:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.296516:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205272.296520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205272.296523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205272.296628:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205272.296634:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205272.296637:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205272.296640:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205272.296644:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205272.296651:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6727 00000400:00000200:2.0:1788205272.296660:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205272.296666:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205272.296669:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205272.296672:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296675:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205272.296677:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296680:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205272.296683:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205272.296685:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205272.296688:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205272.296691:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205272.296694:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205272.296696:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205272.296699:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296700:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205272.296702:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296704:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205272.296707:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205272.296710:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296713:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205272.296716:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205272.296719:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205272.296722:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.296724:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205272.296727:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205272.296736:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000020:00000040:2.0:1788205272.296739:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205272.296743:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205272.296745:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.296748:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205272.296750:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205272.296754:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205272.296762:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d400. 00000100:00000001:2.0:1788205272.296768:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205272.296777:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205272.296788:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205272.296792:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205272.296797:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205272.296845:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205272.296849:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205272.296853:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205272.296854:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205272.296856:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205272.296859:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205272.296862:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6727 00000400:00000200:2.0:1788205272.296867:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205272.296871:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1788205272.296871:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205272.296876:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788205272.296876:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205272.296880:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205272.296881:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296885:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205272.296889:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205272.296890:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296894:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205272.296897:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205272.296898:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205272.296901:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205272.296904:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:1.0:1788205272.296908:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788205272.296909:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:1.0:1788205272.296912:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205272.296914:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:1.0:1788205272.296918:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1788205272.296918:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 02000000:00000001:1.0:1788205272.296922:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205272.296923:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788205272.296928:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205272.296928:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.296933:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00100000:1.0:1788205272.296936:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205945984:192.168.201.151@tcp:250:kworker.0 00000020:00000040:2.0:1788205272.296937:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:1.0:1788205272.296943:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788205272.296944:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:1.0:1788205272.296949:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788205272.296949:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.296953:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205272.296954:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205272.296959:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205272.296961:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1788205272.296964:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bf000. 02000000:00000001:2.0:1788205272.296965:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.296969:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.296970:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205272.296975:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1788205272.296980:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:2.0:1788205272.296981:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.296985:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205272.296986:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000100:00000010:2.0:1788205272.296990:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302d80. 00000020:00000040:2.0:1788205272.296993:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000400:00000010:1.0:1788205272.296994:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 02000000:00000001:2.0:1788205272.296996:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205272.296999:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.297001:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1788205272.297003:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205945984, portal 25 02000000:00000001:2.0:1788205272.297003:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205272.297006:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297008:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205272.297011:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 02000000:00000010:2.0:1788205272.297012:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:2.0:1788205272.297017:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205272.297019:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205272.297026:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205272.297032:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205272.297039:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205272.297042:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1788205272.297044:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000001:2.0:1788205272.297047:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205272.297049:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205945984, offset 0 00000100:00000040:2.0:1788205272.297054:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084489021] to pc [ptlrpcd_rcv+-1] req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205272.297059:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205272.297071:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205272.297079:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205272.297080:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00080000:2.0:1788205272.297082:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205272.297087:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1788205272.297087:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205272.297091:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788205272.297094:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:2.0:1788205272.297094:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205272.297099:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205272.297101:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788205272.297105:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6727 00000400:00000200:1.0:1788205272.297108:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:2.0:1788205272.297114:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.297115:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59246:59246:256:4294967295] 192.168.201.151@tcp LPNI seq info [59226:59226:8:4294967295] 00000400:00000200:2.0:1788205272.297121:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000400:00000200:1.0:1788205272.297123:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:2.0:1788205272.297126:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788205272.297135:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205272.297142:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.297147:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205272.297153:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205272.297157:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205272.297162:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205272.297165:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205272.297169:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.297172:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205272.297178:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205272.297182:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205272.297187:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205272.297190:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205272.297194:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205272.297198:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297207:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.297210:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297217:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297219:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205272.297224:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205272.297227:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205272.297230:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301403649 00000100:00000010:1.0:1788205272.297234:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1ea0. 00000100:00000001:1.0:1788205272.297240:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205272.297246:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301403649 00000100:00000010:1.0:1788205272.297249:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c13c0. 00000100:00000001:1.0:1788205272.297252:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205272.297256:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205272.297258:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297260:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205272.297264:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205272/real 1788205272] req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205272.297272:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000001:1.0:1788205272.297273:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205272.297275:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205272.297278:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000100:00000001:2.0:1788205272.297279:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205272.297283:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000100:00000040:2.0:1788205272.297285:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:1.0:1788205272.297286:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:2.0:1788205272.297290:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788205272.297292:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1788205272.297295:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205272.297298:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000100:00000200:1.0:1788205272.297299:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205272.297301:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205272.297304:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205272.297309:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205272.297313:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000200:1.0:1788205272.297315:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205272.297321:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205272.297327:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205272.297327:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205272.297331:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205272.297331:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205272.297337:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205272.297337:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788205272.297341:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788205272.297342:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205272.297349:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00100000:1.0:1788205272.297350:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205272.297352:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205272.297357:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205272.297387:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205272.297398:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297405:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205272.297405:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205272.297411:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205272.297411:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297416:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205272.297417:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205272.297419:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.297422:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:2.0:1788205272.297422:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000100:00000001:1.0:1788205272.297425:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788205272.297427:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205272.297430:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788205272.297431:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205272.297438:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000010:2.0:1788205272.297439:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301500. 00000100:00000001:1.0:1788205272.297443:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205272.297450:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205272.297454:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:2.0:1788205272.297460:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788205272.297463:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205945984:192.168.201.151@tcp:250:kworker.0 02000000:00000001:2.0:1788205272.297463:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.297468:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205272.297471:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205272.297471:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205272.297475:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788205272.297475:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205272.297482:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000100:00000040:1.0:1788205272.297483:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205945984/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205272.297489:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297492:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205272.297496:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1788205272.297498:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1788205272.297500:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205272.297505:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205272.297508:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205272.297511:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000100:00000001:2.0:1788205272.297511:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788205272.297515:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1788205272.297516:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205272.297519:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d400. 00000100:00000040:2.0:1788205272.297523:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087618bfd] to pc [ptlrpcd_rcv+-1] req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205272.297525:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205272.297528:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205272.297530:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:1.0:1788205272.297533:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205272.297534:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.297536:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.297542:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205272.297543:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297545:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205272.297546:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205272.297547:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205272.297551:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297558:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205272.297559:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.297561:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205272.297563:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205272.297565:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205272.297567:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946112:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205272.297570:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205272.297572:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205272.297574:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.297575:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205272.297578:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205272.297580:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205272.297584:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205272.297586:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205272.297589:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205272.297592:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946112, portal 10 00000100:00000001:1.0:1788205272.297594:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205272.297595:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290478464 : -113511419073152 : ffff98c310302d80) 00000100:00000040:1.0:1788205272.297600:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297607:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205272.297609:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205272.297612:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205272.297614:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205946112, offset 0 00000400:00000200:1.0:1788205272.297617:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205272.297621:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.297624:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205272.297627:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205272.297630:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205272.297632:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205272.297634:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205272.297637:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59247:59247:256:4294967295] 192.168.201.151@tcp LPNI seq info [59227:59227:8:4294967295] 00000400:00000200:1.0:1788205272.297641:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.297646:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205272.297650:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.297653:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205272.297657:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205272.297659:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205272.297663:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205272.297666:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205272.297669:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.297671:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205272.297674:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205272.297677:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205272.297680:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205272.297682:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205272.297684:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205272.297687:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297695:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.297697:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.297716:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297720:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205272.297724:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297725:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205272.297729:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205272.297731:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000100:00080000:2.0:1788205272.297737:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1788205272.297738:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:2.0:1788205272.297740:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205272.297742:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301399707 00000100:00080000:2.0:1788205272.297744:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205272.297746:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301403649 00000100:00000001:2.0:1788205272.297750:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205272.297751:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301399707 00000100:00000001:1.0:1788205272.297756:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301403649 00000100:00080000:2.0:1788205272.297756:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6727 00000100:00000001:1.0:1788205272.297760:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205272.297764:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205272.297765:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205272.297769:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1788205272.297772:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00100000:1.0:1788205272.297776:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205272/real 1788205272] req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205272.297777:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205272.297785:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000001:1.0:1788205272.297790:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788205272.297790:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788205272.297793:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1788205272.297796:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000400:00000010:1.0:1788205272.297797:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000100:00000001:2.0:1788205272.297801:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000400:00000200:1.0:1788205272.297802:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205272.297806:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000020:00000001:2.0:1788205272.297807:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788205272.297810:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1788205272.297811:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205272.297815:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:1.0:1788205272.297817:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205272.297818:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:2.0:1788205272.297833:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:2.0:1788205272.297838:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:2.0:1788205272.297846:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.297851:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1788205272.297852:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205272.297856:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205272.297862:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:1.0:1788205272.297868:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1788205272.297868:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000001:1.0:1788205272.297873:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205272.297874:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205272.297877:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205272.297879:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297880:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205272.297884:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788205272.297888:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205272.297889:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205272.297894:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.297899:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205272.297902:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788205272.297903:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205272.297908:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205272.297909:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205272.297911:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205272.297918:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303480. 00000100:00000001:1.0:1788205272.297920:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1788205272.297924:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:1.0:1788205272.297925:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.297928:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788205272.297930:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205272.297931:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205272.297933:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297935:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205272.297937:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205272.297939:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205272.297941:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1788205272.297943:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788205272.297945:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205272.297951:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 02000000:00000010:2.0:1788205272.297952:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000100:00000001:1.0:1788205272.297955:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788205272.297959:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205272.297962:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205272.297968:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205272.297976:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946112:192.168.201.151@tcp:38:kworker.0 00000100:00000001:2.0:1788205272.297984:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205272.297985:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205272.297988:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.297989:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.297995:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302d80 x1875062205946112/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205272.297998:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4a1f0e9] to pc [ptlrpcd_rcv+-1] req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298008:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205272.298011:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205272.298013:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bf000. 00000100:00000001:2.0:1788205272.298016:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.298017:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205272.298020:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205272.298022:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000100:00000040:2.0:1788205272.298023:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6737) 00000020:00000001:1.0:1788205272.298026:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205272.298029:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:1.0:1788205272.298032:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205272.298033:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205272.298035:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302d80. 00000100:00000001:1.0:1788205272.298038:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205272.298040:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298042:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298046:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205272.298047:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298049:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205272.298051:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205272.298052:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205272.298056:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298061:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205272.298062:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.298064:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205272.298066:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205272.298068:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205272.298070:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946240:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205272.298073:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205272.298075:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205272.298076:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.298078:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205272.298081:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205272.298083:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205272.298086:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205272.298088:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205272.298092:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205272.298094:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946240, portal 4 00000100:00000001:1.0:1788205272.298096:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205272.298097:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290472192 : -113511419079424 : ffff98c310301500) 00000100:00000040:1.0:1788205272.298102:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298108:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205272.298111:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205272.298113:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205272.298115:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205946240, offset 0 00000400:00000200:1.0:1788205272.298118:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205272.298122:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.298125:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205272.298128:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205272.298131:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205272.298133:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205272.298135:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205272.298138:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59248:59248:256:4294967295] 192.168.201.151@tcp LPNI seq info [59228:59228:8:4294967295] 00000400:00000200:1.0:1788205272.298141:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.298147:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205272.298151:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.298154:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205272.298158:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205272.298160:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205272.298164:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205272.298166:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205272.298170:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.298171:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205272.298174:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205272.298177:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205272.298180:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205272.298181:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205272.298183:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205272.298186:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298194:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.298197:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298203:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298205:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205272.298207:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205272.298210:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205272.298213:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205272.298215:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301399593 00000100:00000001:1.0:1788205272.298217:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301403649 00000100:00000001:1.0:1788205272.298219:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301399593 00000100:00000001:1.0:1788205272.298221:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301403649 00000100:00000001:1.0:1788205272.298223:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205272.298224:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298226:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205272.298230:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205272/real 1788205272] req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205272.298236:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205272.298243:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205272.298244:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205272.298246:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205272.298248:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205272.298250:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205272.298253:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205272.298260:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298265:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205272.298267:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298269:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205272.298270:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205272.298273:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298279:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205272.298282:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298288:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205272.298291:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298293:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205272.298294:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298296:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205272.298298:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205272.298300:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205272.298303:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205272.298305:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205272.298308:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205272.298315:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946240:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205272.298318:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205272.298321:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298327:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205272.298329:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.298331:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205272.298332:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205272.298334:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205272.298336:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205272.298339:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205272.298341:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205272.298342:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205272.298343:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205272.298346:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205272.298348:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205272.298351:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205272.298353:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205272.298356:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205272.298358:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946368, portal 4 00000100:00000001:1.0:1788205272.298359:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205272.298361:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290480256 : -113511419071360 : ffff98c310303480) 00000100:00000040:1.0:1788205272.298383:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298389:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205272.298392:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205272.298394:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205272.298396:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205946368, offset 0 00000400:00000200:1.0:1788205272.298400:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205272.298404:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.298407:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205272.298409:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205272.298411:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205272.298413:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205272.298415:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205272.298418:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59249:59249:256:4294967295] 192.168.201.151@tcp LPNI seq info [59229:59229:8:4294967295] 00000400:00000200:1.0:1788205272.298422:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205272.298428:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205272.298432:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.298435:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205272.298439:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205272.298441:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205272.298445:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205272.298448:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205272.298465:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205272.298467:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205272.298470:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205272.298473:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205272.298476:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205272.298478:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205272.298480:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205272.298484:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298490:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.298493:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298498:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298499:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205272.298502:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205272.298505:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205272.298507:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205272.298510:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301399600 00000100:00000001:1.0:1788205272.298512:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301403649 00000100:00000001:1.0:1788205272.298514:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301399600 00000100:00000001:1.0:1788205272.298516:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301403649 00000100:00000001:1.0:1788205272.298518:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205272.298520:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298522:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205272.298525:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205272/real 1788205272] req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205272.298532:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205272.298533:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205272.298535:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205272.298537:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205272.298539:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205272.298541:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205272.298544:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205272.298551:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298557:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205272.298558:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298560:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205272.298561:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205272.298565:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298571:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205272.298575:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298581:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205272.298583:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298585:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205272.298586:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298588:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205272.298589:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205272.298592:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205272.298595:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205272.298597:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205272.298600:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205272.298607:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946368:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205272.298610:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298612:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.298615:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301500 x1875062205946240/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298620:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205272.298621:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205272.298623:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205272.298625:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205272.298626:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205272.298627:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205272.298629:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205272.298632:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:1.0:1788205272.298634:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205272.298635:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205272.298636:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301500. 00000100:00000001:1.0:1788205272.298639:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205272.298640:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298641:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205272.298644:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303480 x1875062205946368/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205272.298649:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205272.298650:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205272.298652:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205272.298653:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205272.298655:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205272.298656:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205272.298657:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205272.298659:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:1.0:1788205272.298660:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205272.298662:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205272.298663:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303480. 00000100:00000001:1.0:1788205272.298665:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205272.298666:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298667:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205272.298670:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205272.298672:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205272.298674:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205272.298676:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205272.488498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205272.488504:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205272.488509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205272.488510:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205272.488513:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205272.488514:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205272.488517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205272.488519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205272.488634:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205272.488638:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205272.488641:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205272.808491:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205272.808498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205272.808500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205272.808512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205272.808516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205272.808522:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205272.808525:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205272.808527:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205272.808530:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205273.256449:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205273.256456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205273.256459:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205273.256464:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205273.256467:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205273.256470:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205273.256473:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205273.256475:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205273.256478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.320468:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205273.320468:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205273.320474:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205273.320475:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.320477:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.320478:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205273.320484:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205273.320485:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205273.320488:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205273.320490:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205273.320491:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205273.320494:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.320494:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205273.320497:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205273.320500:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.320501:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.320504:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.320507:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205273.512464:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205273.512470:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205273.512476:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205273.512477:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205273.512480:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205273.512481:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205273.512485:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205273.512486:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205273.512539:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205273.512542:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205273.512545:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205273.832447:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205273.832455:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.832457:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205273.832463:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205273.832467:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205273.832470:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.832473:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.832476:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.832480:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205274.280465:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205274.280471:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205274.280473:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205274.280478:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205274.280481:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205274.280485:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205274.280487:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205274.280489:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205274.280492:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205274.344454:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205274.344455:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205274.344460:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205274.344462:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205274.344463:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205274.344465:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205274.344468:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205274.344470:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:0.0:1788205274.344471:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205274.344476:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205274.344476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205274.344480:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205274.344481:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205274.344484:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205274.344486:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205274.344487:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205274.344490:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205274.344493:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536518:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205274.536523:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205274.536525:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205274.536527:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536531:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205274.536533:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536534:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536535:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536538:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205274.536539:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536541:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205274.536542:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205274.536543:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205274.536543:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536545:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205274.536546:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536547:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536548:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536550:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205274.536551:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205274.536552:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205274.536553:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536554:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205274.536554:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536555:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205274.536556:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205274.536600:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205274.536603:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205274.536605:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205274.856464:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205274.856471:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205274.856473:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205274.856477:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205274.856480:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205274.856483:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205274.856486:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205274.856488:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205274.856491:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205275.304474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205275.304481:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205275.304484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205275.304489:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205275.304492:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205275.304496:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205275.304499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205275.304501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205275.304504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205275.368461:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205275.368469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205275.368471:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205275.368477:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205275.368480:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205275.368484:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205275.368484:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205275.368489:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205275.368490:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205275.368493:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205275.368495:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205275.368498:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205275.368500:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205275.368504:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205275.368508:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205275.368511:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205275.368513:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205275.368515:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205275.560505:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205275.560512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205275.560517:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205275.560519:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205275.560523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205275.560526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205275.560529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205275.560531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205275.560590:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205275.560594:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205275.560598:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205275.880484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205275.880499:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205275.880501:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205275.880508:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205275.880512:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205275.880517:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205275.880521:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205275.880524:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205275.880527:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205276.328468:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205276.328474:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205276.328475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205276.328486:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205276.328488:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205276.328491:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205276.328493:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205276.328494:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205276.328497:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205276.392457:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205276.392465:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205276.392467:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205276.392472:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205276.392475:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205276.392478:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.392481:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205276.392483:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.392486:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205276.394151:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205276.394154:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205276.394156:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205276.394159:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205276.394161:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205276.394163:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205276.394165:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205276.394166:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205276.394169:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205276.584478:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205276.584484:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205276.584490:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205276.584491:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205276.584494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205276.584495:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205276.584498:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205276.584499:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205276.584552:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205276.584555:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205276.584557:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205276.904453:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301403649, at 4301404160 00000100:00000040:1.0:1788205276.904453:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301403649, at 4301404160 00000100:00000001:3.0:1788205276.904453:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205276.904460:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205276.904464:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205276.904467:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1ea0. 00000100:00100000:3.0:1788205276.904468:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000010:1.0:1788205276.904471:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c13c0. 00000100:00000001:3.0:1788205276.904474:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205276.904487:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.904490:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205276.904492:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.904494:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.352466:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205277.352473:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205277.352475:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205277.352481:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205277.352484:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205277.352488:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.352491:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205277.352493:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.352496:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205277.416476:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205277.416478:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205277.416479:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205277.416485:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205277.416485:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788205277.416487:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205277.416490:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205277.416490:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416491:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1788205277.416497:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00100000:3.0:1788205277.416497:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00080000:2.0:1788205277.416498:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205277.416505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205277.416505:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205277.416506:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205277.416511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205277.416512:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205277.416516:0:12771:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6732 00000100:00000001:3.0:1788205277.416517:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205277.416520:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205277.416523:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205277.416524:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205277.416527:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205277.416529:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205277.416531:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205277.416537:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205277.416542:0:12771:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205277.416546:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416550:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205277.416554:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416556:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205277.416560:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205277.416561:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205277.416564:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205277.416566:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205277.416567:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205277.416569:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205277.416571:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416572:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205277.416573:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416575:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205277.416577:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205277.416579:0:12771:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416580:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205277.416582:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205277.416584:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205277.416586:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416588:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205277.416589:0:12771:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205277.416596:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302680. 00000020:00000040:2.0:1788205277.416598:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205277.416601:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205277.416603:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416605:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205277.416606:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205277.416609:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205277.416616:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0d000. 00000100:00000001:2.0:1788205277.416620:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205277.416626:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205277.416633:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205277.416636:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205277.416641:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3cb18a5] to pc [ptlrpcd_rcv+-1] req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205277.416662:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205277.416665:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205277.416668:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205277.416670:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205277.416671:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205277.416674:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205277.416677:0:12771:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6732 00000400:00000200:2.0:1788205277.416681:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205277.416683:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205277.416685:0:12771:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205277.416687:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416689:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205277.416691:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416693:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205277.416695:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205277.416696:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205277.416698:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205277.416700:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205277.416702:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:1.0:1788205277.416703:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205277.416704:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:1.0:1788205277.416708:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205277.416711:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205277.416712:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205277.416717:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205277.416722:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205277.416722:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416726:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205277.416749:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:1.0:1788205277.416750:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205277.416753:0:12771:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205277.416754:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416756:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205277.416759:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205277.416760:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205277.416764:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205277.416765:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205277.416768:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205277.416769:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416773:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1788205277.416775:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205946496:192.168.201.151@tcp:250:kworker.0 00000008:00100000:2.0:1788205277.416778:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205277.416780:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788205277.416783:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.416785:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788205277.416786:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.416789:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205277.416792:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310303b80. 02000000:00000001:1.0:1788205277.416793:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1788205277.416797:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000010:1.0:1788205277.416800:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:2.0:1788205277.416801:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205277.416805:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.416806:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416807:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205277.416810:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205277.416812:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.416816:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 02000000:00000010:2.0:1788205277.416818:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0dc00. 00000400:00000010:1.0:1788205277.416821:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000100:00000001:2.0:1788205277.416823:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205277.416829:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788205277.416830:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000001:2.0:1788205277.416838:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1788205277.416839:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946496, portal 25 00000100:00000001:1.0:1788205277.416843:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205277.416846:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290476672 : -113511419074944 : ffff98c310302680) 00000100:00000001:2.0:1788205277.416846:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205277.416852:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ec2b05d] to pc [ptlrpcd_rcv+-1] req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205277.416857:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205277.416861:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205277.416866:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205277.416872:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205277.416876:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205277.416877:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788205277.416877:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205277.416881:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1788205277.416883:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000100:00080000:2.0:1788205277.416886:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6732 00000400:00000010:1.0:1788205277.416888:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000400:00000200:2.0:1788205277.416892:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000200:1.0:1788205277.416894:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205946496, offset 0 00000400:00000200:2.0:1788205277.416896:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205277.416900:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205277.416903:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000400:00000200:1.0:1788205277.416907:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416907:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205277.416912:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416913:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205277.416915:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205277.416916:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205277.416918:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1788205277.416919:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000020:00000040:2.0:1788205277.416921:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205277.416924:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205277.416926:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:1.0:1788205277.416929:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000040:2.0:1788205277.416931:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416935:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205277.416938:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788205277.416938:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416943:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205277.416945:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000400:00000200:1.0:1788205277.416948:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:2.0:1788205277.416950:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205277.416953:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.416956:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1788205277.416956:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205277.416960:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205277.416961:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 02000000:00000001:2.0:1788205277.416963:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.416966:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205277.416969:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59250:59250:256:4294967295] 192.168.201.151@tcp LPNI seq info [59230:59230:8:4294967295] 00000008:00100000:2.0:1788205277.416970:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000400:00000200:1.0:1788205277.416977:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:2.0:1788205277.416982:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.416989:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000100:00000001:2.0:1788205277.416990:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205277.416997:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000800:00000200:1.0:1788205277.416998:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000020:00000040:2.0:1788205277.417001:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000800:00000010:1.0:1788205277.417005:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 02000000:00000001:2.0:1788205277.417006:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205277.417010:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.417012:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1788205277.417015:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 02000000:00000001:2.0:1788205277.417015:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205277.417019:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205277.417021:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 02000000:00000010:2.0:1788205277.417025:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0c400. 00000800:00000200:1.1:1788205277.417030:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:2.0:1788205277.417031:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205277.417035:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000100:00100000:2.0:1788205277.417041:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205277.417045:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.417049:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000100:00000001:2.0:1788205277.417057:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205277.417058:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205277.417062:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.417066:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788205277.417072:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205277.417074:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205277.417078:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205277.417083:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205277.417088:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205277.417100:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417104:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788205277.417105:0:12771:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205277.417112:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205277.417112:0:12771:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205277.417116:0:12771:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205277.417118:0:12771:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205277.417126:0:12771:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205277.417129:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205277.417132:0:12771:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6732 00000100:00000001:1.0:1788205277.417133:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205277.417139:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205277.417139:0:12771:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.417146:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205277.417147:0:12771:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205277.417150:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301408769 00000100:00080000:2.0:1788205277.417151:0:12771:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000010:1.0:1788205277.417156:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c13c0. 00000100:00000040:2.0:1788205277.417157:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.417163:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205277.417165:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205277.417167:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788205277.417169:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205277.417173:0:12771:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788205277.417174:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301408769 00000020:00000040:2.0:1788205277.417174:0:12771:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:2.0:1788205277.417179:0:12771:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000010:1.0:1788205277.417181:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1120. 00000020:00000040:2.0:1788205277.417183:0:12771:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:1.0:1788205277.417185:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788205277.417188:0:12771:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:1.0:1788205277.417190:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788205277.417192:0:12771:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:1.0:1788205277.417194:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417198:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788205277.417199:0:12771:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.417204:0:12771:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788205277.417205:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205277/real 1788205277] req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205277.417209:0:12771:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205277.417215:0:12771:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205277.417220:0:12771:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000400:00000001:1.0:1788205277.417223:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788205277.417224:0:12771:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000400:00000001:1.0:1788205277.417225:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205277.417229:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000100:00000001:2.0:1788205277.417230:0:12771:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.417232:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 02000000:00000001:2.0:1788205277.417234:0:12771:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1788205277.417236:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 02000000:00000001:2.0:1788205277.417239:0:12771:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205277.417240:0:2345:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1788205277.417244:0:12771:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205277.417246:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205277.417248:0:12771:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205277.417252:0:12771:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:2.0:1788205277.417257:0:12771:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205277.417262:0:12771:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205277.417264:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788205277.417268:0:12771:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205277.417273:0:12771:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:1.0:1788205277.417277:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788205277.417278:0:12771:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205277.417280:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205277.417282:0:12771:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417283:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205277.417286:0:12771:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205277.417287:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205277.417290:0:12771:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788205277.417294:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205277.417294:0:12771:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205277.417302:0:12771:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0e800. 00000100:00000001:1.0:1788205277.417306:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205277.417308:0:12771:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205277.417313:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205277.417318:0:12771:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205277.417327:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205277.417332:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417336:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205277.417337:0:12771:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205277.417338:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205277.417342:0:12771:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417343:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205277.417347:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788205277.417351:0:12771:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205277.417352:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205277.417359:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205277.417385:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205277.417396:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205277.417396:0:12771:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205277.417404:0:12771:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6742) 00000100:00100000:1.0:1788205277.417410:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205946496:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205277.417415:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417418:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.417421:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302680 x1875062205946496/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417428:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205277.417430:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205277.417433:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205277.417436:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205277.417438:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205277.417439:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205277.417441:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205277.417444:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0d000. 02000000:00000001:1.0:1788205277.417447:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205277.417449:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205277.417450:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302680. 00000100:00000001:1.0:1788205277.417453:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205277.417455:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417457:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417463:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205277.417465:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417467:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205277.417468:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205277.417470:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205277.417473:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417480:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205277.417482:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.417484:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205277.417485:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205277.417487:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205277.417490:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946624:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205277.417494:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205277.417495:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205277.417497:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.417499:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205277.417502:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205277.417503:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.417507:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205277.417510:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205277.417513:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205277.417515:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946624, portal 10 00000100:00000001:1.0:1788205277.417518:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205277.417519:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290482048 : -113511419069568 : ffff98c310303b80) 00000100:00000040:1.0:1788205277.417525:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417532:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205277.417534:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205277.417537:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205277.417539:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205946624, offset 0 00000400:00000200:1.0:1788205277.417542:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205277.417546:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.417550:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205277.417552:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205277.417555:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205277.417557:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205277.417560:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205277.417563:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59251:59251:256:4294967295] 192.168.201.151@tcp LPNI seq info [59231:59231:8:4294967295] 00000400:00000200:1.0:1788205277.417567:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.417573:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205277.417577:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.417581:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205277.417585:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205277.417588:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205277.417592:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205277.417594:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205277.417597:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.417599:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205277.417602:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205277.417606:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205277.417609:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205277.417610:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205277.417613:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205277.417616:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417630:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.417632:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417639:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417640:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205277.417643:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205277.417646:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205277.417649:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205277.417652:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301404827 00000100:00000001:1.0:1788205277.417654:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.417656:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301404827 00000100:00000001:1.0:1788205277.417658:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.417661:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205277.417663:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417664:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205277.417668:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205277/real 1788205277] req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205277.417676:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205277.417678:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205277.417679:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205277.417681:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205277.417683:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205277.417685:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205277.417688:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205277.417697:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417703:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205277.417705:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417707:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.417708:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205277.417712:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417718:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205277.417722:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417728:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205277.417742:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417744:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.417745:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417746:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205277.417748:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205277.417751:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205277.417754:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205277.417756:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205277.417761:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205277.417770:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310303b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946624:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205277.417774:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205277.417777:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417784:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205277.417785:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.417787:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205277.417789:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205277.417791:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205277.417794:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946752:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205277.417797:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205277.417799:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205277.417800:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.417802:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205277.417805:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205277.417807:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.417810:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205277.417812:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205277.417816:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205277.417818:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946752, portal 4 00000100:00000001:1.0:1788205277.417820:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205277.417821:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:1.0:1788205277.417826:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417833:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205277.417835:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205277.417837:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205277.417839:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205946752, offset 0 00000400:00000200:1.0:1788205277.417843:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205277.417846:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.417849:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205277.417852:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205277.417854:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205277.417857:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205277.417859:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205277.417862:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59252:59252:256:4294967295] 192.168.201.151@tcp LPNI seq info [59232:59232:8:4294967295] 00000400:00000200:1.0:1788205277.417867:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.417873:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205277.417877:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.417880:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205277.417884:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205277.417887:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205277.417891:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205277.417893:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205277.417897:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.417899:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205277.417902:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205277.417905:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205277.417907:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205277.417909:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205277.417911:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205277.417915:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417922:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.417925:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417932:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417934:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205277.417936:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205277.417939:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205277.417942:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205277.417944:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301404713 00000100:00000001:1.0:1788205277.417946:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.417949:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301404713 00000100:00000001:1.0:1788205277.417951:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.417953:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205277.417955:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.417957:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205277.417961:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205277/real 1788205277] req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205277.417968:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205277.417970:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205277.417971:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205277.417973:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205277.417975:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205277.417977:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205277.417980:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205277.417989:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.417995:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205277.417996:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.417998:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.417999:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205277.418003:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418009:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205277.418013:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418020:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205277.418022:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418023:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.418025:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418026:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205277.418028:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205277.418030:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205277.418033:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205277.418035:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205277.418038:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205277.418052:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946752:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205277.418056:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205277.418059:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418066:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205277.418067:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.418069:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205277.418071:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205277.418073:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205277.418075:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205277.418078:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205277.418080:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205277.418081:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205277.418083:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205277.418086:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205277.418087:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205277.418090:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205277.418092:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205277.418095:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205277.418098:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205946880, portal 4 00000100:00000001:1.0:1788205277.418099:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205277.418101:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:1.0:1788205277.418106:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418112:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205277.418114:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205277.418117:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205277.418119:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205946880, offset 0 00000400:00000200:1.0:1788205277.418122:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205277.418126:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.418129:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205277.418132:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205277.418134:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205277.418136:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205277.418139:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205277.418141:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59253:59253:256:4294967295] 192.168.201.151@tcp LPNI seq info [59233:59233:8:4294967295] 00000400:00000200:1.0:1788205277.418145:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205277.418152:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205277.418156:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.418159:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205277.418162:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205277.418165:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205277.418169:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205277.418172:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205277.418175:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205277.418177:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205277.418179:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205277.418183:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205277.418185:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205277.418187:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205277.418189:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205277.418192:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418199:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.418202:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418208:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.418210:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205277.418212:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205277.418215:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205277.418218:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205277.418221:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301404719 00000100:00000001:1.0:1788205277.418223:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.418225:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301404719 00000100:00000001:1.0:1788205277.418227:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301408769 00000100:00000001:1.0:1788205277.418229:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205277.418231:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.418233:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205277.418236:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205277/real 1788205277] req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205277.418243:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205277.418245:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205277.418246:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205277.418248:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205277.418250:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205277.418252:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205277.418255:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205277.418263:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418269:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205277.418271:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418272:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.418274:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205277.418277:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418284:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205277.418287:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418294:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205277.418296:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418298:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205277.418299:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418301:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205277.418303:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205277.418305:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205277.418308:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205277.418310:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205277.418314:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205277.418321:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205946880:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205277.418325:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418326:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.418330:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310303b80 x1875062205946624/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418336:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205277.418337:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205277.418339:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bd400. 02000000:00000001:1.0:1788205277.418341:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205277.418343:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205277.418344:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205277.418346:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205277.418348:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0dc00. 02000000:00000001:1.0:1788205277.418350:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205277.418351:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205277.418353:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310303b80. 00000100:00000001:1.0:1788205277.418355:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205277.418356:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418358:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.418361:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205946752/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418389:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205277.418390:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205277.418392:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bf000. 02000000:00000001:1.0:1788205277.418394:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205277.418395:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205277.418396:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205277.418398:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205277.418400:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0c400. 02000000:00000001:1.0:1788205277.418402:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205277.418403:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205277.418404:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:1.0:1788205277.418406:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205277.418408:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418409:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205277.418413:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205946880/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205277.418419:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205277.418420:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205277.418422:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bc800. 02000000:00000001:1.0:1788205277.418424:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205277.418425:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205277.418426:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205277.418428:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205277.418429:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0e800. 02000000:00000001:1.0:1788205277.418431:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205277.418432:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205277.418433:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:1.0:1788205277.418435:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205277.418436:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418438:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205277.418441:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205277.418442:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205277.418444:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205277.418445:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205277.608509:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205277.608515:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205277.608521:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205277.608523:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205277.608525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205277.608527:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205277.608529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205277.608531:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205277.608595:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205277.608598:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205277.608601:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205277.932250:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205277.932257:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205277.932259:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205277.932264:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205277.932267:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205277.932278:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205277.932285:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205277.932288:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205277.932291:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205278.376484:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205278.376493:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205278.376495:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205278.376500:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205278.376504:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205278.376508:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205278.376510:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205278.376513:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205278.376516:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205278.440450:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205278.440457:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205278.440460:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205278.440465:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205278.440469:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205278.440472:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205278.440475:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205278.440477:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205278.440479:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205278.440557:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205278.440560:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205278.440561:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205278.440564:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205278.440566:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205278.440568:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.440570:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205278.440571:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.440573:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205278.632512:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205278.632518:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205278.632523:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205278.632525:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205278.632528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205278.632529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205278.632532:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205278.632533:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205278.632874:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205278.632878:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205278.632880:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205278.952693:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205278.952701:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205278.952703:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205278.952736:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205278.952740:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205278.952744:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.952747:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205278.952750:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.952752:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.400474:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205279.400482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205279.400484:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205279.400490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205279.400493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205279.400497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.400499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205279.400502:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.400505:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205279.464534:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205279.464534:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205279.464546:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205279.464547:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205279.464553:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205279.464554:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.464567:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00100000:0.0:1788205279.464568:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:3.0:1788205279.464580:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205279.464581:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205279.464591:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.464591:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.464601:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.464606:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.464610:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205279.464693:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205279.464707:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205279.464711:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205279.656536:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205279.656543:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205279.656549:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205279.656551:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205279.656554:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205279.656557:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205279.656559:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205279.656561:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205279.657148:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205279.657151:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205279.657155:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205279.980360:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205279.980416:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205279.980419:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.980427:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205279.980431:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205279.980437:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.980440:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.980447:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.980451:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205280.424726:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205280.424734:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205280.424736:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205280.424741:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205280.424744:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205280.424748:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205280.424751:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205280.424753:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205280.424757:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205280.488441:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205280.488447:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205280.488449:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205280.488465:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205280.488469:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205280.488472:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205280.488476:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205280.488478:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205280.488481:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205280.490103:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205280.490111:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205280.490113:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205280.490118:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205280.490122:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205280.490126:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205280.490129:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205280.490131:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205280.490134:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205280.680709:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205280.680718:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205280.680726:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205280.680727:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205280.680731:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205280.680732:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205280.680736:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205280.680737:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205280.681357:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205280.681362:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205280.681422:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205281.000490:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205281.000498:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205281.000500:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205281.000506:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205281.000509:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205281.000513:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.000516:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205281.000520:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.000522:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205281.448448:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205281.448456:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205281.448457:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205281.448463:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205281.448466:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205281.448469:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205281.448472:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205281.448475:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205281.448478:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205281.512463:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205281.512469:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205281.512470:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205281.512474:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205281.512476:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205281.512479:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.512481:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205281.512483:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.512484:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205281.512697:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205281.512700:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205281.512701:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205281.512704:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205281.512706:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205281.512708:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205281.512710:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205281.512712:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205281.512713:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704494:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205281.704500:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205281.704509:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205281.704512:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205281.704515:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704518:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205281.704520:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704522:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704525:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704528:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205281.704529:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205281.704533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205281.704535:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205281.704594:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205281.704597:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205281.704600:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788205282.024491:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301408769, at 4301409280 00000100:00000040:1.0:1788205282.024491:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301408769, at 4301409280 00000100:00000010:0.0:1788205282.024507:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c13c0. 00000100:00000010:1.0:1788205282.024509:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c3202c1120. 00000100:00000001:3.0:1788205282.024578:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205282.024583:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205282.024585:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205282.024589:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205282.024592:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205282.024595:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205282.024598:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205282.024600:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205282.024603:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.472462:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205282.472467:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205282.472469:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205282.472472:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205282.472475:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205282.472478:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205282.472480:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205282.472481:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205282.472483:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205282.536414:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205282.536420:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205282.536423:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205282.536428:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205282.536432:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205282.536436:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205282.536439:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205282.536440:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205282.536442:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.536445:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205282.536447:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.536448:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205282.536454:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205282.536457:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205282.536460:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.536462:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205282.536464:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.536467:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205282.536472:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205282.536479:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205282.536482:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205282.536486:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205282.536489:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205282.536496:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6737 00000400:00000200:1.0:1788205282.536504:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205282.536510:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205282.536514:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205282.536516:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536520:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205282.536522:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536525:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:1.0:1788205282.536528:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205282.536530:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:1.0:1788205282.536532:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205282.536535:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:1.0:1788205282.536538:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:1.0:1788205282.536540:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:1.0:1788205282.536543:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536545:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205282.536547:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536549:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205282.536552:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:1.0:1788205282.536555:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536558:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205282.536560:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205282.536563:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205282.536566:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.536568:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205282.536570:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205282.536578:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534a80. 00000020:00000040:1.0:1788205282.536582:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:1.0:1788205282.536585:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205282.536587:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.536590:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205282.536593:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205282.536597:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205282.536604:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bc800. 00000100:00000001:1.0:1788205282.536610:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205282.536628:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205282.536638:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205282.536670:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205282.536676:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4e81942] to pc [ptlrpcd_rcv+-1] req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205282.536697:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205282.536700:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205282.536704:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205282.536705:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205282.536706:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205282.536710:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205282.536713:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6737 00000400:00000200:1.0:1788205282.536717:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205282.536721:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00000001:0.0:1788205282.536724:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788205282.536724:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205282.536729:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205282.536729:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205282.536733:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205282.536733:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205282.536738:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536742:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000040:0.0:1788205282.536743:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205282.536746:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205282.536749:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:1.0:1788205282.536753:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205282.536756:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000100:00000001:0.0:1788205282.536758:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1788205282.536760:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000001:0.0:1788205282.536761:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205282.536765:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 02000000:00000001:0.0:1788205282.536767:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205282.536771:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205282.536771:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536776:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788205282.536777:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205282.536780:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00100000:0.0:1788205282.536785:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205947008:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205282.536786:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205282.536790:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00000001:0.0:1788205282.536793:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1788205282.536797:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 02000000:00000001:0.0:1788205282.536798:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788205282.536801:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.536802:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205282.536804:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205282.536807:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205282.536812:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1788205282.536813:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:1.0:1788205282.536816:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.536817:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.536819:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205282.536825:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1788205282.536826:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000008:00000001:1.0:1788205282.536831:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1788205282.536833:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000100:00000001:1.0:1788205282.536834:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205282.536841:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534e00. 00000400:00000010:0.0:1788205282.536843:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000020:00000040:1.0:1788205282.536844:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:1.0:1788205282.536850:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205282.536851:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205282.536853:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947008, portal 25 00000100:00000001:1.0:1788205282.536853:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205282.536856:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788205282.536858:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1788205282.536859:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.536861:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292779648 : -113511416771968 : ffff98c310534a80) 02000000:00000010:1.0:1788205282.536865:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bf000. 00000100:00000040:0.0:1788205282.536870:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205282.536871:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205282.536881:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205282.536882:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205282.536890:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205282.536896:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000001:1.0:1788205282.536896:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205282.536900:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205282.536902:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205947008, offset 0 00000100:00000040:1.0:1788205282.536906:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a26415e2] to pc [ptlrpcd_rcv+-1] req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205282.536913:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000100:00000001:1.0:1788205282.536923:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205282.536924:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00080000:1.0:1788205282.536927:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205282.536931:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:0.0:1788205282.536932:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000100:00000001:1.0:1788205282.536934:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205282.536938:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1788205282.536940:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205282.536942:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1788205282.536947:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000100:00080000:1.0:1788205282.536948:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6737 00000400:00000200:0.0:1788205282.536964:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205282.536967:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205282.536971:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59254:59254:256:4294967295] 192.168.201.151@tcp LPNI seq info [59234:59234:8:4294967295] 00000400:00000200:0.0:1788205282.536975:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.537045:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205282.537052:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205282.537054:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000800:00000010:0.0:1788205282.537059:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000400:00000200:1.0:1788205282.537060:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205282.537065:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205282.537070:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000800:00000200:0.0:1788205282.537071:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000100:00000001:1.0:1788205282.537073:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1788205282.537076:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1788205282.537077:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000800:00000200:0.1:1788205282.537084:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205282.537087:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788205282.537091:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.537093:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205282.537099:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205282.537103:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205282.537108:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205282.537110:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205282.537114:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205282.537118:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537128:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.537131:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205282.537140:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205282.537143:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205282.537144:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788205282.537147:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1788205282.537148:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:1.0:1788205282.537152:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1788205282.537154:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000100:00000001:0.0:1788205282.537156:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205282.537157:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:1.0:1788205282.537161:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000400:00000200:0.0:1788205282.537165:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000040:1.0:1788205282.537169:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205282.537171:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301413889 00000100:00000001:1.0:1788205282.537173:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205282.537177:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000010:0.0:1788205282.537179:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9c00. 00000100:00000001:1.0:1788205282.537179:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205282.537185:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:1.0:1788205282.537187:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:0.0:1788205282.537189:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205282.537190:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205282.537194:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788205282.537198:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301413889 02000000:00000001:1.0:1788205282.537198:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205282.537201:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.537203:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205282.537205:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:1.0:1788205282.537207:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205282.537214:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c307ff9ea0. 00000100:00000001:0.0:1788205282.537218:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205282.537219:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537222:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1788205282.537225:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310536d80. 00000100:00000001:0.0:1788205282.537227:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205282.537230:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 00000100:00000001:0.0:1788205282.537232:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788205282.537234:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205282.537236:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.537238:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1788205282.537241:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205282/real 1788205282] req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205282.537241:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205282.537243:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205282.537248:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bd400. 00000100:00000001:1.0:1788205282.537254:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788205282.537257:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205282.537261:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:1.0:1788205282.537262:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1788205282.537264:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205282.537269:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205282.537271:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205282.537274:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205282.537279:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205282.537300:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205282.537303:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788205282.537304:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205282.537309:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3ea5fa4] to pc [ptlrpcd_rcv+-1] req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537325:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205282.537325:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537329:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205282.537329:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788205282.537335:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1788205282.537335:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205282.537338:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788205282.537339:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205282.537340:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205282.537345:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205282.537348:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6737 00000100:00100000:0.0:1788205282.537349:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205282.537353:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205282.537357:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205282.537361:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788205282.537384:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205282.537390:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205282.537398:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.537402:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788205282.537405:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1788205282.537406:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.537409:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000100:00000001:0.0:1788205282.537412:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788205282.537412:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1788205282.537414:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000100:00000001:0.0:1788205282.537416:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1788205282.537417:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788205282.537420:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788205282.537421:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000100:00000001:0.0:1788205282.537425:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:1.0:1788205282.537425:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000001:0.0:1788205282.537429:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1788205282.537430:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00080000:0.0:1788205282.537436:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1788205282.537437:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:1.0:1788205282.537441:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1788205282.537443:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000040:1.0:1788205282.537446:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:0.0:1788205282.537449:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788205282.537450:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:1.0:1788205282.537455:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00000040:0.0:1788205282.537457:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205282.537461:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:1.0:1788205282.537466:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205282.537471:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205282.537475:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788205282.537478:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205947008:192.168.201.151@tcp:250:kworker.0 02000000:00000001:1.0:1788205282.537479:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205282.537483:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205282.537487:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1788205282.537487:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:1.0:1788205282.537491:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537492:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205282.537494:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205282.537498:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534a80 x1875062205947008/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788205282.537500:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310534700. 00000020:00000040:1.0:1788205282.537505:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 00000100:00000001:0.0:1788205282.537512:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205282.537515:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205282.537516:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788205282.537519:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205282.537520:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 00000100:00000001:1.0:1788205282.537522:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205282.537525:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1788205282.537525:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205282.537527:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205282.537529:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205282.537532:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 02000000:00000010:1.0:1788205282.537533:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c3102bdc00. 00000020:00000001:0.0:1788205282.537537:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1788205282.537539:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205282.537541:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bc800. 02000000:00000001:0.0:1788205282.537546:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:1.0:1788205282.537547:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788205282.537549:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205282.537553:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534a80. 00000100:00000001:0.0:1788205282.537558:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205282.537561:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205282.537562:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205282.537566:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537567:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205282.537573:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d809345a] to pc [ptlrpcd_rcv+-1] req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537576:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205282.537579:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537583:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205282.537587:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537588:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205282.537591:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205282.537595:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6747) 00000100:00000040:0.0:1788205282.537598:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537606:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205282.537607:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.537623:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205282.537625:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205282.537627:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205282.537630:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947136:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205282.537633:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205282.537635:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205282.537636:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.537637:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205282.537640:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205282.537655:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205282.537660:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205282.537663:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205282.537666:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205282.537669:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947136, portal 10 00000100:00000001:0.0:1788205282.537671:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205282.537672:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292780544 : -113511416771072 : ffff98c310534e00) 00000100:00000040:0.0:1788205282.537678:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537684:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205282.537686:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205282.537689:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205282.537691:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205947136, offset 0 00000400:00000200:0.0:1788205282.537695:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205282.537698:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.537703:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205282.537705:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205282.537708:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205282.537710:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205282.537712:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205282.537715:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59255:59255:256:4294967295] 192.168.201.151@tcp LPNI seq info [59235:59235:8:4294967295] 00000400:00000200:0.0:1788205282.537719:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.537724:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205282.537729:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.537732:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205282.537736:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205282.537739:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788205282.537744:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205282.537746:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788205282.537749:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.537751:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205282.537755:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205282.537758:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205282.537761:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205282.537762:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205282.537765:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205282.537769:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537776:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.537779:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537785:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537786:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205282.537789:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205282.537792:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:0.0:1788205282.537795:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205282.537797:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301409947 00000100:00000001:0.0:1788205282.537800:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.537802:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301409947 00000100:00000001:0.0:1788205282.537804:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.537806:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205282.537808:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.537810:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205282.537814:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205282/real 1788205282] req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205282.537821:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205282.537822:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205282.537823:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205282.537826:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205282.537827:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205282.537829:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205282.537832:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205282.537839:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537845:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205282.537846:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.537848:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.537849:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205282.537853:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537859:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205282.537862:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537869:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205282.537872:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.537873:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.537874:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.537876:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205282.537878:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205282.537881:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205282.537884:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205282.537886:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205282.537890:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205282.537897:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947136:192.168.201.151@tcp:38:kworker.0 00000100:00000001:0.0:1788205282.537901:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205282.537904:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537910:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205282.537912:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.537914:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205282.537915:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205282.537917:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205282.537920:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947264:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205282.537922:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205282.537924:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205282.537926:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.537928:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205282.537931:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205282.537933:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205282.537937:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205282.537939:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205282.537942:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205282.537945:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947264, portal 4 00000100:00000001:0.0:1788205282.537947:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205282.537948:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292788608 : -113511416763008 : ffff98c310536d80) 00000100:00000040:0.0:1788205282.537953:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.537960:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205282.537962:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205282.537965:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205282.537967:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205947264, offset 0 00000400:00000200:0.0:1788205282.537971:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205282.537974:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.537978:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205282.537992:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205282.537995:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205282.537997:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205282.537999:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205282.538002:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59256:59256:256:4294967295] 192.168.201.151@tcp LPNI seq info [59236:59236:8:4294967295] 00000400:00000200:0.0:1788205282.538006:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.538012:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205282.538017:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.538020:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205282.538024:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205282.538027:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788205282.538031:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205282.538034:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788205282.538037:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.538039:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205282.538042:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205282.538046:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205282.538049:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205282.538050:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205282.538053:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205282.538056:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538063:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.538067:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538073:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.538074:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205282.538077:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205282.538080:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:0.0:1788205282.538083:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205282.538085:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301409833 00000100:00000001:0.0:1788205282.538087:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.538089:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301409833 00000100:00000001:0.0:1788205282.538091:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.538093:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205282.538094:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.538096:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205282.538100:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205282/real 1788205282] req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205282.538107:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205282.538108:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205282.538109:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205282.538112:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205282.538114:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205282.538116:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205282.538119:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205282.538127:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538132:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205282.538134:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538135:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.538137:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205282.538141:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538147:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205282.538150:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538157:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205282.538159:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538161:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.538162:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538163:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205282.538165:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205282.538168:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205282.538171:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205282.538173:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205282.538176:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205282.538184:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310536d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947264:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205282.538187:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788205282.538190:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538196:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205282.538198:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.538200:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205282.538201:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205282.538203:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205282.538206:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205282.538209:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205282.538211:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205282.538212:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205282.538214:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788205282.538217:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205282.538219:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205282.538222:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:0.0:1788205282.538225:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c3202da210. 00000400:00000010:0.0:1788205282.538228:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217e58. 00000100:00000200:0.0:1788205282.538230:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947392, portal 4 00000100:00000001:0.0:1788205282.538232:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205282.538233:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562292778752 : -113511416772864 : ffff98c310534700) 00000100:00000040:0.0:1788205282.538238:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538245:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205282.538247:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:0.0:1788205282.538249:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c319217220. 00000100:00000200:0.0:1788205282.538252:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205947392, offset 0 00000400:00000200:0.0:1788205282.538255:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:0.0:1788205282.538258:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.538262:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:0.0:1788205282.538265:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788205282.538267:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:0.0:1788205282.538270:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205282.538272:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:0.0:1788205282.538275:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59257:59257:256:4294967295] 192.168.201.151@tcp LPNI seq info [59237:59237:8:4294967295] 00000400:00000200:0.0:1788205282.538278:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:0.0:1788205282.538284:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:0.0:1788205282.538288:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.538291:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c3054af200. 00000800:00000200:0.0:1788205282.538294:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.0:1788205282.538297:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788205282.538301:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:0.1:1788205282.538303:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788205282.538307:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:0.0:1788205282.538309:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c3054af200. 00000400:00000200:0.0:1788205282.538312:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788205282.538316:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788205282.538318:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217220 00000400:00000010:0.0:1788205282.538320:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217220. 00000100:00000001:0.0:1788205282.538322:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205282.538326:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538332:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.538335:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538341:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.538342:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205282.538345:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205282.538348:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:0.0:1788205282.538351:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:0.0:1788205282.538353:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301409839 00000100:00000001:0.0:1788205282.538355:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.538357:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301409839 00000100:00000001:0.0:1788205282.538359:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301413889 00000100:00000001:0.0:1788205282.538361:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205282.538395:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.538397:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205282.538401:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205282/real 1788205282] req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788205282.538409:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788205282.538410:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788205282.538412:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c3202da210. 00000400:00000200:0.0:1788205282.538415:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c319217e58 00000400:00000010:0.0:1788205282.538417:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c319217e58. 00000100:00000001:0.0:1788205282.538419:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788205282.538422:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205282.538430:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538436:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205282.538437:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538439:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.538440:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788205282.538443:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538450:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205282.538453:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538459:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205282.538462:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538463:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205282.538464:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538466:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205282.538468:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205282.538471:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205282.538474:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:0.0:1788205282.538476:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788205282.538480:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205282.538488:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310534700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947392:192.168.201.151@tcp:8:kworker.0 00000100:00000001:0.0:1788205282.538491:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538493:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.538497:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534e00 x1875062205947136/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538503:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205282.538505:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205282.538506:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dca000. 02000000:00000001:0.0:1788205282.538509:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205282.538511:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205282.538512:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:0.0:1788205282.538514:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205282.538515:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bf000. 02000000:00000001:0.0:1788205282.538518:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205282.538519:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205282.538521:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534e00. 00000100:00000001:0.0:1788205282.538523:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205282.538524:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538526:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.538529:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310536d80 x1875062205947264/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538535:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205282.538537:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205282.538538:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcb000. 02000000:00000001:0.0:1788205282.538540:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205282.538541:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205282.538542:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:0.0:1788205282.538544:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205282.538545:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bd400. 02000000:00000001:0.0:1788205282.538547:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205282.538548:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205282.538549:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310536d80. 00000100:00000001:0.0:1788205282.538552:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205282.538553:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538555:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205282.538557:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310534700 x1875062205947392/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205282.538563:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788205282.538565:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205282.538566:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c310dcac00. 02000000:00000001:0.0:1788205282.538568:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788205282.538569:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788205282.538570:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:0.0:1788205282.538572:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788205282.538573:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c3102bdc00. 02000000:00000001:0.0:1788205282.538575:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788205282.538576:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788205282.538577:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310534700. 00000100:00000001:0.0:1788205282.538579:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205282.538580:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538582:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205282.538585:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205282.538587:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205282.538588:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205282.538590:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205282.728790:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205282.728797:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205282.728805:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205282.728806:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205282.728809:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205282.728811:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205282.728814:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205282.728815:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205282.732749:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205282.732757:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205282.732761:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205283.048484:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205283.048491:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205283.048492:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205283.048498:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205283.048501:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205283.048505:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205283.048508:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205283.048511:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205283.048513:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205283.496752:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205283.496759:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205283.496762:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205283.496766:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205283.496770:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205283.496774:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205283.496776:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205283.496778:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205283.496781:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205283.560487:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205283.560494:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205283.560496:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205283.560501:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205283.560505:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205283.560509:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205283.560512:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205283.560514:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205283.560517:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205283.560608:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205283.560611:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205283.560612:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205283.560616:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205283.560618:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205283.560620:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205283.560621:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205283.560623:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205283.560660:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205283.752482:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205283.752488:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205283.752493:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205283.752494:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205283.752497:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205283.752498:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205283.752510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205283.752512:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205283.752815:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205283.752819:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205283.752822:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205284.072499:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205284.072506:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205284.072508:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.072512:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205284.072516:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205284.072519:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.072522:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.072525:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.072528:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.520478:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205284.520485:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205284.520488:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.520493:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205284.520497:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205284.520501:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205284.520503:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205284.520506:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205284.520508:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.584481:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205284.584489:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205284.584491:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.584496:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205284.584500:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205284.584504:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.584506:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.584508:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.584512:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205284.584602:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205284.584605:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205284.584636:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205284.584638:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205284.584640:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205284.584642:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205284.584645:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205284.584646:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205284.584648:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776535:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205284.776542:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205284.776544:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205284.776548:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776552:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205284.776554:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776556:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776559:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776562:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205284.776564:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776567:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205284.776569:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205284.776570:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205284.776571:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776573:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205284.776574:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776582:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776584:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776587:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205284.776588:0:48498:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1788205284.776590:0:48498:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1788205284.776591:0:48498:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776592:0:48498:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1788205284.776594:0:48498:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776595:0:48498:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205284.776597:0:48498:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205284.776698:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205284.776701:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205284.776703:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205285.096442:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205285.096451:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205285.096453:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205285.096458:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205285.096462:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205285.096466:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205285.096469:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205285.096472:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205285.096475:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205285.544475:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205285.544482:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205285.544485:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205285.544490:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205285.544493:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205285.544497:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205285.544499:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205285.544501:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205285.544504:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205285.608489:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205285.608497:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205285.608499:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205285.608504:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205285.608507:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205285.608511:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205285.608514:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205285.608516:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205285.608519:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205285.619958:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205285.619967:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205285.619969:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205285.619974:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205285.619977:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205285.619981:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205285.619984:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205285.619987:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205285.619990:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205285.800676:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205285.800682:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205285.800687:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205285.800688:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205285.800692:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205285.800693:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205285.800696:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205285.800698:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205285.800807:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205285.800811:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205285.800815:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205286.120481:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205286.120488:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205286.120491:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205286.120496:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205286.120500:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205286.120504:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205286.120507:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205286.120509:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205286.120512:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205286.568541:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205286.568548:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205286.568550:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205286.568555:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205286.568559:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205286.568562:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205286.568565:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205286.568567:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205286.568570:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205286.632474:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205286.632482:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205286.632484:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205286.632488:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205286.632492:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205286.632496:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205286.632498:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205286.632501:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205286.632504:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205286.632674:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205286.632678:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205286.632679:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205286.632683:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205286.632685:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205286.632687:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205286.632689:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205286.632690:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205286.632693:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205286.824514:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205286.824520:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205286.824525:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205286.824526:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205286.824529:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205286.824530:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205286.824533:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205286.824534:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205286.824632:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205286.824635:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205286.824637:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1788205287.144485:0:48498:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda110 (cpu 1) has reached its deadline 4301413889, at 4301414400 00000100:00000010:1.0:1788205287.144500:0:48498:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9ea0. 00000100:00000040:0.0:1788205287.144887:0:48526:0:(connection.c:75:cpu_latency_work()) work item of ffff98c318cda000 (cpu 0) has reached its deadline 4301413889, at 4301414400 00000100:00000010:0.0:1788205287.144894:0:48526:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c307ff9c00. 00000100:00000001:3.0:1788205287.145269:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205287.145273:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205287.145275:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205287.145279:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:3.0:1788205287.145282:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205287.145285:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.145288:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205287.145290:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.145293:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.592421:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205287.592427:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.592428:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.592432:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205287.592435:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.592438:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.592439:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.592441:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.592443:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205287.660740:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205287.660748:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205287.660751:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205287.660756:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205287.660759:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205287.660763:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205287.660766:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205287.660769:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205287.660772:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.661170:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205287.661173:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205287.661174:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205287.661177:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205287.661180:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205287.661181:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.661183:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205287.661184:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.661186:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205287.663825:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8c827723-576e-478f-8d68-06b4dd922260->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205287.663835:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205287.663838:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205287.663841:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205287.663846:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205287.663852:0:30130:0:(import.c:510:import_select_connection()) MGC192.168.201.151@tcp: connect to NID 192.168.201.151@tcp last attempt 6742 00000400:00000200:2.0:1788205287.663861:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205287.663866:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205287.663870:0:30130:0:(import.c:589:import_select_connection()) MGC192.168.201.151@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205287.663873:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664075:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.664081:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664084:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205287.664089:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205287.664091:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64249 00000020:00000001:2.0:1788205287.664094:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205287.664097:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7e800 refcount=3 00000020:00000001:2.0:1788205287.664099:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000020:00000001:2.0:1788205287.664101:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169350144 : -113511540201472 : ffff98c308f7e800) 00000100:00000040:2.0:1788205287.664104:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664106:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.664108:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664110:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205287.664113:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7e800 : new refcount 2 00000100:00080000:2.0:1788205287.664116:0:30130:0:(import.c:624:import_select_connection()) MGC192.168.201.151@tcp: import ffff98c3114c4800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664119:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205287.664122:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205287.664126:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205287.664128:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.664131:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205287.664133:0:30130:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205287.664142:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310301c00. 00000020:00000040:2.0:1788205287.664146:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c3114c4800 refcount=4 obd=MGC192.168.201.151@tcp 02000000:00000001:2.0:1788205287.664149:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205287.664151:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.664154:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205287.664157:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205287.664161:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205287.664169:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0cc00. 00000100:00000001:2.0:1788205287.664176:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205287.664185:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205287.664195:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205287.664200:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205287.664204:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040d392e4] to pc [ptlrpcd_rcv+-1] req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.664258:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.664263:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.664265:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205287.664272:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.664279:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.664281:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.664295:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205287.664298:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205287.664300:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.664303:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205947520:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205287.664310:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205287.664314:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205287.664316:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.664318:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205287.664323:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.664326:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205287.664332:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205287.664336:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205287.664342:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205287.664348:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947520, portal 25 00000100:00000001:1.0:1788205287.664351:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205287.664352:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290473984 : -113511419077632 : ffff98c310301c00) 00000100:00000040:1.0:1788205287.664358:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.664380:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205287.664383:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205287.664386:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205287.664389:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875062205947520, offset 0 00000400:00000200:1.0:1788205287.664395:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205287.664403:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.664409:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205287.664414:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205287.664418:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205287.664423:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205287.664425:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205287.664429:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59258:59258:256:4294967295] 192.168.201.151@tcp LPNI seq info [59238:59238:8:4294967295] 00000400:00000200:1.0:1788205287.664434:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.664442:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205287.664448:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.664452:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205287.664459:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205287.664463:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205287.664469:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205287.664473:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205287.664487:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.664490:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205287.664497:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205287.664502:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205287.664507:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205287.664509:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205287.664512:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205287.664516:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205287.664680:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205287.664684:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205287.664688:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205287.664689:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205287.664691:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205287.664694:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205287.664696:0:30130:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6742 00000400:00000200:2.0:1788205287.664701:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205287.664704:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205287.664707:0:30130:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205287.664710:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664712:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.664714:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664716:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205287.664719:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205287.664720:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d64296 00000020:00000001:2.0:1788205287.664722:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205287.664724:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c304df3000 refcount=5 00000020:00000001:2.0:1788205287.664726:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000020:00000001:2.0:1788205287.664728:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562100621312 : -113511608930304 : ffff98c304df3000) 00000100:00000040:2.0:1788205287.664730:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664732:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.664745:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664747:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205287.664750:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c304df3000 : new refcount 4 00000100:00080000:2.0:1788205287.664752:0:30130:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff98c32021e000: import ffff98c307f67000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205287.664756:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205287.664758:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205287.664764:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205287.664766:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.664768:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205287.664772:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205287.664775:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.664777:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205287.664782:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310302300. 00000020:00000040:2.0:1788205287.664784:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f67000 refcount=203 obd=lustre-MDT0000-mdc-ffff98c32021e000 02000000:00000001:2.0:1788205287.664786:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205287.664788:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.664790:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205287.664791:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205287.664793:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205287.664797:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c305a0ec00. 00000100:00000001:2.0:1788205287.664801:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205287.664806:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205287.664812:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205287.664815:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205287.664818:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058d8cae2] to pc [ptlrpcd_rcv+-1] req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665203:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.665210:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665218:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665220:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205287.665227:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205287.665231:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205287.665234:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 0 is 4301419017 00000100:00000010:1.0:1788205287.665239:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c1120. 00000100:00000001:1.0:1788205287.665245:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788205287.665253:0:2345:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS new deadline estimation for cpu 1 is 4301419017 00000100:00000010:1.0:1788205287.665256:0:2345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c3202c14e0. 00000100:00000001:1.0:1788205287.665259:0:2345:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.151@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788205287.665261:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205287.665264:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665267:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205287.665272:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205287/real 1788205287] req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205287.665292:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205287.665294:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205287.665296:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205287.665299:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205287.665300:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205287.665303:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205287.665307:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205287.665316:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665323:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205287.665325:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665327:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.665328:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205287.665332:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665339:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205287.665344:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665351:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205287.665354:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665355:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.665356:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665358:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205287.665360:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205287.665377:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c3114c4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205287.665383:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205287.665385:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205287.665390:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205287.665397:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8c827723-576e-478f-8d68-06b4dd922260:2345:1875062205947520:192.168.201.151@tcp:250:kworker.0 00000100:00000001:1.0:1788205287.665403:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665406:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.665409:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310301c00 x1875062205947520/t0(0) o250->MGC192.168.201.151@tcp@192.168.201.151@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665416:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205287.665419:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205287.665421:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.665424:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205287.665426:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205287.665428:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c3114c4800 refcount=3 obd=MGC192.168.201.151@tcp 00000020:00000001:1.0:1788205287.665430:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205287.665432:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0cc00. 02000000:00000001:1.0:1788205287.665436:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205287.665438:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205287.665440:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310301c00. 00000100:00000001:1.0:1788205287.665443:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205287.665444:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665447:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665454:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.665456:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665458:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.665460:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.665461:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205287.665465:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665471:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.665473:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.665475:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205287.665594:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205287.665597:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.665602:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947648:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205287.665605:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205287.665607:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205287.665609:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.665611:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205287.665615:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.665617:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205287.665621:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205287.665624:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205287.665628:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205287.665632:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947648, portal 10 00000100:00000001:1.0:1788205287.665634:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205287.665635:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562290475776 : -113511419075840 : ffff98c310302300) 00000100:00000040:1.0:1788205287.665642:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665648:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205287.665650:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205287.665652:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205287.665654:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875062205947648, offset 0 00000400:00000200:1.0:1788205287.665659:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205287.665665:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.665669:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205287.665673:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205287.665677:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205287.665679:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205287.665682:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205287.665685:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59259:59259:256:4294967295] 192.168.201.151@tcp LPNI seq info [59239:59239:8:4294967295] 00000400:00000200:1.0:1788205287.665690:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.665698:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205287.665703:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.665707:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205287.665712:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205287.665715:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205287.665719:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205287.665722:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205287.665726:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.665728:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205287.665733:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205287.665737:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205287.665740:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205287.665742:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205287.665744:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205287.665748:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665755:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.665758:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665765:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665767:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205287.665770:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205287.665773:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: using a timeout of 153165 usecs (154 jiffies) 00000400:00000200:1.0:1788205287.665776:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205287.665778:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301415074 00000100:00000001:1.0:1788205287.665780:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.665782:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301415074 00000100:00000001:1.0:1788205287.665784:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.665786:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205287.665788:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665790:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205287.665794:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205287/real 1788205287] req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205287.665801:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205287.665802:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205287.665804:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205287.665807:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205287.665808:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205287.665810:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205287.665814:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205287.665821:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665826:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205287.665828:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665829:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.665830:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205287.665834:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665839:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205287.665843:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665849:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205287.665851:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665853:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.665854:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665856:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205287.665859:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205287.665861:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f67000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205287.665864:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205287.665867:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205287.665870:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205287.665941:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310302300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947648:192.168.201.151@tcp:38:kworker.0 00000100:00000001:1.0:1788205287.665945:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665947:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.665950:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310302300 x1875062205947648/t0(0) o38->lustre-MDT0000-mdc-ffff98c32021e000@192.168.201.151@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.665956:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205287.665958:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205287.665960:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.665963:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205287.665964:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205287.665966:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f67000 refcount=202 obd=lustre-MDT0000-mdc-ffff98c32021e000 00000020:00000001:1.0:1788205287.665967:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205287.665969:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c305a0ec00. 02000000:00000001:1.0:1788205287.665971:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205287.665973:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205287.665974:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310302300. 00000100:00000001:1.0:1788205287.665977:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205287.665978:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665980:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.665983:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.665984:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.665986:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.666053:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.666144:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.666146:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205287.666237:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205287.666241:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205287.666245:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205287.666247:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205287.666249:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205287.666252:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205287.666255:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6742 00000400:00000200:2.0:1788205287.666260:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:2.0:1788205287.666263:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205287.666266:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205287.666269:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.666272:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.666275:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.666277:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:2.0:1788205287.666280:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1788205287.666342:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642a4 00000020:00000001:2.0:1788205287.666345:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1788205287.666348:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c308f7c000 refcount=4 00000020:00000001:2.0:1788205287.666350:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000020:00000001:2.0:1788205287.666353:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562169339904 : -113511540211712 : ffff98c308f7c000) 00000100:00000040:2.0:1788205287.666356:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.666358:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205287.666360:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:2.0:1788205287.666362:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:2.0:1788205287.666419:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c308f7c000 : new refcount 3 00000100:00080000:2.0:1788205287.666423:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff98c32021e000: import ffff98c307f66000 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:2.0:1788205287.666425:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205287.666427:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205287.666600:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205287.666604:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.666606:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205287.666609:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 3407872, lost: 0. 00000008:00000001:3.0:1788205287.666612:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.666614:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205287.666620:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce2a00. 00000020:00000040:3.0:1788205287.666623:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66000 refcount=22 obd=lustre-OST0000-osc-ffff98c32021e000 02000000:00000001:3.0:1788205287.666626:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205287.666628:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.666631:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205287.666633:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205287.666636:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205287.666642:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c23000. 00000100:00000001:3.0:1788205287.666647:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205287.666654:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205287.666662:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205287.666666:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205287.666671:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c42bbc] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.666717:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.666720:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.666721:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205287.666726:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.666732:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.666733:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.666735:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205287.666737:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205287.666739:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.666742:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205287.666746:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205287.666748:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205287.666750:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.666752:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205287.666756:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.666757:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205287.666762:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205287.666765:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205287.666770:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205287.666774:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947776, portal 4 00000100:00000001:1.0:1788205287.666776:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205287.666777:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300832256 : -113511408719360 : ffff98c310ce2a00) 00000100:00000040:1.0:1788205287.666783:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.666789:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205287.666791:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205287.666793:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205287.666795:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205947776, offset 0 00000400:00000200:1.0:1788205287.666801:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205287.666806:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.666811:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205287.666815:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205287.666818:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205287.666821:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205287.666823:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205287.666826:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59260:59260:256:4294967295] 192.168.201.151@tcp LPNI seq info [59240:59240:8:4294967295] 00000400:00000200:1.0:1788205287.666830:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.666837:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205287.666842:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.666847:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205287.666852:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205287.666854:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205287.666859:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205287.666861:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205287.666864:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.666866:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205287.666871:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205287.666874:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205287.666949:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205287.666952:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205287.666955:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205287.666959:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205287.666980:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205287.666983:0:30130:0:(pinger.c:229:ptlrpc_pinger_process_import()) a2c88ea5-0f5e-47e2-a306-099c37c6f2a6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205287.666987:0:30130:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205287.666997:0:30130:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205287.666999:0:30130:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205287.667001:0:30130:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205287.667004:0:30130:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: connect to NID 192.168.201.151@tcp last attempt 6742 00000400:00000200:3.0:1788205287.667008:0:30130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:3.0:1788205287.667011:0:30130:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.151@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205287.667013:0:30130:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205287.667016:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205287.667018:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205287.667020:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205287.667022:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000001:3.0:1788205287.667024:0:30130:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1788205287.667025:0:30130:0:(genops.c:804:class_conn2export()) looking for export cookie 0x3b83dfecb3d642ab 00000020:00000001:3.0:1788205287.667027:0:30130:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1788205287.667029:0:30130:0:(lustre_handles.c:148:class_handle2object()) GET export ffff98c318d19000 refcount=4 00000020:00000001:3.0:1788205287.667031:0:30130:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000020:00000001:3.0:1788205287.667033:0:30130:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446630562435272704 : -113511274278912 : ffff98c318d19000) 00000100:00000040:3.0:1788205287.667035:0:30130:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff98c303f7ad80 refcount 11 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205287.667037:0:30130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205287.667038:0:30130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff98c303f7ad80 refcount 12 to 192.168.201.151@tcp 00000100:00000001:3.0:1788205287.667039:0:30130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630562085449088 : -113511624102528 : ffff98c303f7ad80) 00000020:00000040:3.0:1788205287.667041:0:30130:0:(genops.c:879:class_export_put()) PUTting export ffff98c318d19000 : new refcount 3 00000100:00080000:3.0:1788205287.667044:0:30130:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff98c32021e000: import ffff98c307f66800 using connection 192.168.201.151@tcp/192.168.201.151@tcp 00000100:00000001:3.0:1788205287.667046:0:30130:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205287.667048:0:30130:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205287.667051:0:30130:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205287.667053:0:30130:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.667055:0:30130:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205287.667058:0:30130:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 1703936, lost: 0. 00000008:00000001:3.0:1788205287.667060:0:30130:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.667061:0:30130:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205287.667065:0:30130:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c310ce0a80. 00000020:00000040:3.0:1788205287.667067:0:30130:0:(genops.c:1210:class_import_get()) import ffff98c307f66800 refcount=14 obd=lustre-OST0001-osc-ffff98c32021e000 02000000:00000001:3.0:1788205287.667070:0:30130:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205287.667071:0:30130:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.667072:0:30130:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205287.667074:0:30130:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205287.667075:0:30130:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205287.667079:0:30130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff98c310c22800. 00000100:00000001:3.0:1788205287.667082:0:30130:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205287.667086:0:30130:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205287.667092:0:30130:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205287.667094:0:30130:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205287.667097:0:30130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed2b742] to pc [ptlrpcd_rcv+-1] req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667444:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.667451:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667459:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.667460:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205287.667467:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205287.667471:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: using a timeout of 39366 usecs (40 jiffies) 00000400:00000200:1.0:1788205287.667475:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205287.667551:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301414963 00000100:00000001:1.0:1788205287.667563:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.667565:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301414963 00000100:00000001:1.0:1788205287.667567:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.667570:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205287.667572:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.667575:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205287.667580:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205287/real 1788205287] req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205287.667591:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205287.667592:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205287.667594:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205287.667598:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205287.667599:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205287.667602:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205287.667606:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205287.667614:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667621:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205287.667623:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667625:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.667627:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205287.667631:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667637:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205287.667641:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667647:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205287.667650:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667651:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.667653:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667655:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205287.667658:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205287.667661:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205287.667665:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205287.667667:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205287.667671:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205287.667678:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947776:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205287.667682:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667684:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.667687:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce2a00 x1875062205947776/t0(0) o8->lustre-OST0000-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667699:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205287.667701:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205287.667703:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.667706:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205287.667708:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205287.667710:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66000 refcount=21 obd=lustre-OST0000-osc-ffff98c32021e000 00000020:00000001:1.0:1788205287.667712:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205287.667713:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c23000. 02000000:00000001:1.0:1788205287.667716:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205287.667718:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205287.667720:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce2a00. 00000100:00000001:1.0:1788205287.667723:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205287.667724:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667727:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.667733:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.667735:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.667737:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.667738:0:2345:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.667740:0:2345:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205287.667744:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667751:0:2345:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.667753:0:2345:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.667755:0:2345:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205287.667757:0:2345:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205287.667759:0:2345:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.667763:0:2345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205287.667766:0:2345:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205287.667768:0:2345:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205287.667770:0:2345:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205287.667771:0:2345:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205287.667776:0:2345:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.667777:0:2345:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205287.667781:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000010:1.0:1788205287.667784:0:2345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c307e3b3c8. 00000400:00000010:1.0:1788205287.667787:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657f68. 00000100:00000200:1.0:1788205287.667791:0:2345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875062205947904, portal 4 00000100:00000001:1.0:1788205287.667793:0:2345:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205287.667794:0:2345:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446630562300824192 : -113511408727424 : ffff98c310ce0a80) 00000100:00000040:1.0:1788205287.667800:0:2345:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.667806:0:2345:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205287.667808:0:2345:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.151@tcp 00000400:00000010:1.0:1788205287.667811:0:2345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c318657dd0. 00000100:00000200:1.0:1788205287.667813:0:2345:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875062205947904, offset 0 00000400:00000200:1.0:1788205287.667819:0:2345:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.151@tcp 00000400:00000200:1.0:1788205287.667824:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.667830:0:2345:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.151@tcp local destination 00000400:00000200:1.0:1788205287.667834:0:2345:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205287.667838:0:2345:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.51@tcp 00000400:00000200:1.0:1788205287.667841:0:2345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.51@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205287.667844:0:2345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.151@tcp 00000400:00000200:1.0:1788205287.667848:0:2345:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.51@tcp NI seq info: [59261:59261:256:4294967295] 192.168.201.151@tcp LPNI seq info [59241:59241:8:4294967295] 00000400:00000200:1.0:1788205287.667852:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000400:00000200:1.0:1788205287.667859:0:2345:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.51@tcp(192.168.201.51@tcp:) -> 192.168.201.151@tcp(192.168.201.151@tcp:192.168.201.151@tcp) : PUT try# 0 00000800:00000200:1.0:1788205287.667864:0:2345:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.667869:0:2345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c319264600. 00000800:00000200:1.0:1788205287.667875:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.0:1788205287.667963:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205287.667969:0:2345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c302eb3cc0] -> 12345-192.168.201.151@tcp (2) 00000800:00000200:1.1:1788205287.667973:0:2345:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.151 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205287.667977:0:2345:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.151@tcp 00000800:00000010:1.0:1788205287.667979:0:2345:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c319264600. 00000400:00000200:1.0:1788205287.667983:0:2345:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205287.668058:0:2345:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.51@tcp->192.168.201.151@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205287.668063:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657dd0 00000400:00000010:1.0:1788205287.668065:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657dd0. 00000100:00000001:1.0:1788205287.668069:0:2345:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205287.668074:0:2345:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668083:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.668086:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205312 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668092:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.668094:0:2345:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205287.668098:0:2345:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205287.668101:0:2345:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: using a timeout of 45152 usecs (46 jiffies) 00000400:00000200:1.0:1788205287.668104:0:2345:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.151@tcp to cpt 0 00000100:00000001:1.0:1788205287.668106:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 0 is 4301414969 00000100:00000001:1.0:1788205287.668109:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 0, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.668111:0:2345:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS new deadline estimation for cpu 1 is 4301414969 00000100:00000001:1.0:1788205287.668113:0:2345:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff98c32021e000: PM QoS request active for cpu 1, keep current deadline 4301419017 00000100:00000001:1.0:1788205287.668114:0:2345:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205287.668117:0:2345:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.668118:0:2345:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205287.668122:0:2345:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205287/real 1788205287] req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205287.668129:0:2345:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205287.668131:0:2345:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205287.668132:0:2345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c307e3b3c8. 00000400:00000200:1.0:1788205287.668135:0:2345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c318657f68 00000400:00000010:1.0:1788205287.668136:0:2345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c318657f68. 00000100:00000001:1.0:1788205287.668138:0:2345:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205287.668142:0:2345:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205287.668150:0:2345:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668161:0:2345:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205287.668163:0:2345:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668165:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.668166:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205287.668169:0:2345:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668176:0:2345:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205287.668180:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668186:0:2345:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205287.668189:0:2345:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668191:0:2345:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205287.668192:0:2345:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668194:0:2345:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205287.668196:0:2345:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205287.668198:0:2345:0:(import.c:65:import_set_state_nolock()) ffff98c307f66800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205287.668201:0:2345:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.151@tcp failed (-110) 00000100:00000001:1.0:1788205287.668203:0:2345:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205287.668207:0:2345:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205287.668215:0:2345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff98c310ce0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a2c88ea5-0f5e-47e2-a306-099c37c6f2a6:2345:1875062205947904:192.168.201.151@tcp:8:kworker.0 00000100:00000001:1.0:1788205287.668219:0:2345:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668221:0:2345:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205287.668224:0:2345:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff98c310ce0a80 x1875062205947904/t0(0) o8->lustre-OST0001-osc-ffff98c32021e000@192.168.201.151@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205312 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205287.668229:0:2345:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205287.668231:0:2345:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205287.668234:0:2345:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98c3102bcc00. 02000000:00000001:1.0:1788205287.668236:0:2345:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205287.668238:0:2345:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1788205287.668239:0:2345:0:(genops.c:1223:class_import_put()) import ffff98c307f66800 refcount=13 obd=lustre-OST0001-osc-ffff98c32021e000 00000020:00000001:1.0:1788205287.668241:0:2345:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1788205287.668243:0:2345:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff98c310c22800. 02000000:00000001:1.0:1788205287.668246:0:2345:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205287.668247:0:2345:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205287.668248:0:2345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c310ce0a80. 00000100:00000001:1.0:1788205287.668251:0:2345:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205287.668252:0:2345:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668254:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.668258:0:2345:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.668259:0:2345:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205287.668261:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205287.668262:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205287.668654:0:2345:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205287.668658:0:2345:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205287.668750:0:30130:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205287.668758:0:30130:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6752) 00010000:00000001:1.0:1788205287.851423:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205287.851432:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205287.851438:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205287.851440:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205287.851444:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205287.851445:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205287.851449:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205287.851450:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205287.851913:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205287.851917:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205287.851920:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205288.169315:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205288.169321:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205288.169323:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205288.169328:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205288.169330:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205288.169334:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205288.169336:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205288.169339:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205288.169341:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205288.616646:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205288.616653:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205288.616655:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205288.616660:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205288.616663:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205288.616667:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205288.616670:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205288.616673:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205288.616675:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205288.680532:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205288.680553:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205288.680555:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205288.680562:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205288.680567:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205288.680572:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205288.680575:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205288.680579:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205288.680582:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205288.681746:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205288.681753:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205288.681755:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205288.681760:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205288.681764:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205288.681767:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205288.681770:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205288.681773:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205288.681775:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205288.872713:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205288.872721:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205288.872726:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205288.872727:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205288.872730:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205288.872732:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205288.872735:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205288.872736:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205288.873644:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205288.873650:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205288.873654:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205289.193747:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205289.193755:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205289.193757:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.193763:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205289.193767:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205289.193771:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205289.193773:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205289.193776:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205289.193779:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205289.640482:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205289.640489:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205289.640491:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205289.640497:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205289.640500:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205289.640503:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205289.640506:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205289.640508:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205289.640511:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.704507:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205289.704517:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205289.704551:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.704557:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205289.704561:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205289.704566:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205289.704569:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205289.704572:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205289.704575:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.704697:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205289.704702:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205289.704703:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.704706:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205289.704709:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205289.704711:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.704713:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.704715:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.704717:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205289.896489:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205289.896496:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205289.896502:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205289.896503:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205289.896506:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205289.896507:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205289.896510:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205289.896511:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205289.896580:0:16070:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205289.896585:0:16070:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205289.896590:0:16070:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205290.216536:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205290.216546:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205290.216549:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205290.216555:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205290.216560:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205290.216564:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205290.216567:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205290.216571:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205290.216573:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205290.664538:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205290.664545:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205290.664547:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205290.664552:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205290.664555:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205290.664559:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205290.664562:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205290.664564:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205290.664567:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205290.728529:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205290.728539:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205290.728541:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205290.728548:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205290.728553:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205290.728558:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205290.728561:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205290.728564:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205290.728568:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205290.728674:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205290.728677:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205290.728679:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205290.728682:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205290.728684:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205290.728686:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205290.728688:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205290.728689:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205290.728692:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205290.944058:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205290.944064:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205290.944070:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205290.944071:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205290.944074:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205290.944075:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1788205290.944078:0:48498:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1788205290.944079:0:48498:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205290.944180:0:36444:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205290.944187:0:36444:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205290.944192:0:36444:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788205291.240602:0:47418:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205291.240612:0:47418:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205291.240615:0:47418:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205291.240621:0:47418:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c319244600 going to sleep for 0 seconds 00000100:00000001:0.0:1788205291.240626:0:47418:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205291.240631:0:47418:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205291.240635:0:47418:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205291.240638:0:47418:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205291.240641:0:47418:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205291.688609:0:48026:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205291.688616:0:48026:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205291.688618:0:48026:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205291.688624:0:48026:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c307f0fb40 going to sleep for 0 seconds 00000100:00000001:1.0:1788205291.688628:0:48026:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205291.688632:0:48026:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205291.688634:0:48026:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205291.688636:0:48026:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205291.688639:0:48026:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205291.752559:0:47415:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205291.752570:0:47415:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205291.752572:0:47415:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205291.752579:0:47415:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920900 going to sleep for 0 seconds 00000100:00000001:3.0:1788205291.752584:0:47415:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205291.752588:0:47415:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205291.752592:0:47415:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205291.752595:0:47415:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205291.752599:0:47415:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205291.752718:0:47417:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788205291.752722:0:47417:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205291.752723:0:47417:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205291.752726:0:47417:0:(client.c:2715:ptlrpc_set_wait()) set ffff98c309920a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788205291.752729:0:47417:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205291.752731:0:47417:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205291.752733:0:47417:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205291.752734:0:47417:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205291.752816:0:47417:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) Debug log: 132618 lines, 132618 kept, 0 dropped, 0 bad.